package bench import ( "bytes" "context" "database/sql" "encoding/json" "fmt" "math" "net/url" "os" "regexp" "strings" "time" _ "github.com/go-sql-driver/mysql" host "github.com/pingcap/endless/pkg/workloadnode/tiup" "github.com/pingcap/log" "github.com/pingcap/test-infra/sdk/resource" _ "github.com/pingcap/test-infra/sdk/resource/impl/k8s" "github.com/pingcap/endless/pkg/workloadnode" backoff "github.com/cenkalti/backoff/v4" "github.com/pelletier/go-toml/v2" "github.com/prometheus/client_golang/api" v1 "github.com/prometheus/client_golang/api/prometheus/v1" "github.com/prometheus/common/model" "go.uber.org/zap" "gopkg.in/yaml.v3" ) func deployCluster(tc *host.TiUPCluster, version string, ignoreConfigCheck bool) error { deployBackOff := backoff.NewExponentialBackOff() deployBackOff.InitialInterval = 5 * time.Second deployBackOff.MaxInterval = 3 * time.Minute deployBackOff.Multiplier = 3 deployBackOff.MaxElapsedTime = 5 * time.Minute retryable := func() error { var err error if ignoreConfigCheck { err = tc.Deploy(version, "--ignore-config-check", "true", "--format", "json", "-c", "2") } else { err = tc.Deploy(version, "--format", "json", "-c", "2") } if err != nil { log.Info("deploy error: ", zap.Error(err)) return err } return nil } err := backoff.Retry(retryable, deployBackOff) return err } func jdbcEscape(benchType, jdbc string) string { if benchType == "corebanking" || benchType == "hzbank" || benchType == "booking" { return strings.Replace(jdbc, "&", "\\&", -1) } else if benchType == "benchmarksql" { return strings.Replace(jdbc, "&", "\\&", -1) } return jdbc } func uploadProfile(workload resource.WorkloadNode, ctx context.Context, version, benchType, benchSubType, thread, dsn, testbedName string, startTime, endTime int64) { db, err := sql.Open("mysql", dsn) if err != nil { panic(err) } defer db.Close() if startTime < endTime-1800 { startTime = endTime - 1800 } filename := fmt.Sprintf("%s_%s_%s_%s_%d.zip", version, benchType, benchSubType, thread, startTime) profileCmd := workloadnode.CommandContext(ctx, fmt.Sprintf(profileCmdTpl, startTime, endTime, filename)) profileCmd.Run(workload) cpProfileCmd := workloadnode.CommandContext(ctx, fmt.Sprintf(cpProfileCmdTpl, filename, testbedName, filename)) cpProfileCmd.Run(workload) workloadHost := host.NewWorkloadHost(workload) uploadOutput, err := workloadHost.Exec(fmt.Sprintf(genProfileURLCmdTpl, testbedName, filename), 10*time.Minute) if err != nil { log.Error("Exec error: ", zap.Error(err)) return } var re = regexp.MustCompile(`.*(http://[^,]+)`) group := re.FindStringSubmatch(uploadOutput) if len(group) != 2 { log.Info("no download url in the output") return } url := group[1] for _, t := range getType(benchType) { updateStmt := fmt.Sprintf(updateProfileUrl, t, strings.TrimSpace(url), testbedName) log.Info(updateStmt) _, err1 := db.Exec(updateStmt) if err1 != nil { log.Error("Save profile url error: ", zap.Error(err)) } } } func uploadClinicWrapper(tc *host.TiUPCluster, benchType, dsn, testbedName, startTime, arch string, isMonitor, isBisect bool) { db, err := sql.Open("mysql", dsn) if err != nil { panic(err) } defer db.Close() flag := "--exclude" updateStmtTemplate := updateLogUrl if isMonitor && !isBisect { flag = "--include" updateStmtTemplate = updateMonitorUrl } if isMonitor && isBisect { flag = "--include" updateStmtTemplate = updateMonitorUrlBisect } url := uploadClinic(tc, flag, "monitor", "-f", startTime) typeList := getType(benchType) log.Info(url) for _, t := range typeList { updateStmt := fmt.Sprintf(updateStmtTemplate, t, url, arch, testbedName) log.Info(updateStmt) _, err := db.Exec(updateStmt) if err != nil { log.Error("Save clinic url error: ", zap.Error(err)) } } } func getType(benchType string) []string { if benchType == "ycsb" || benchType == "sysbench" || benchType == "tpcc" || benchType == "benchmarksql" || benchType == "bank" || benchType == "shenma" || benchType == "corebanking" || benchType == "booking" { return []string{"oltp"} } else if benchType == "tpch" || benchType == "tpcds" || benchType == "job" || benchType == "apdogfood" { return []string{"olap"} } else { return []string{"oltp", "olap"} } } func uploadClinic(tc *host.TiUPCluster, flags ...string) string { url, _ := tc.Diag().WithUploadMaxSize(10 * 1024 * 1024 * 1024).CollectAndUpload(host.DiagCollectAndUploadOptions{ Collect: host.DiagCollectOptions{ Flags: flags}, Upload: host.DiagUploadOptions{ Token: "eyJrIjoiSHBlVjFTdnI1cHlDMG5PeSIsInUiOjMyLCJpZCI6MzN9", Flags: []string{"--rebuild=false"}}}) return url } func checkPlugin(config string) (bool, string, string) { configMap, err := JsonToMap([]byte(config)) if err != nil { return false, "", "" } if _, ok := configMap["plugin"]; ok { map1 := configMap["plugin"].(map[string]interface{}) load, ok2 := map1["load"] dir, ok3 := map1["dir"] if ok2 && ok3 { return true, load.(string), dir.(string) } return false, "", "" } return false, "", "" } func handlePlugins(load, dir, urlPrefix string, tidbReplicas int) (bool, string) { cmdList := []string{} sshList := []string{} if len(urlPrefix) > 0 { for _, v := range strings.Split(load, ",") { cmdList = append(cmdList, fmt.Sprintf(pluginCmdTpl, dir, dir, strings.TrimSuffix(urlPrefix, "/"), v)) } cmd := strings.Join(cmdList, " && ") // we can not tiup cluster exec due to https://github.com/pingcap/tiup/issues/1790 for i := 1; i <= tidbReplicas; i++ { hostname := fmt.Sprintf("tidb-%d-peer", i) sshList = append(sshList, fmt.Sprintf(pluginSSHCmdTpl, hostname, cmd)) } return true, strings.Join(sshList, " && ") } return false, "" } func processConfig(config, benchType, componentType string) (string, error) { // default configration // tidb: mem-quota-query // tikv: storage.block-cache.capacity": 30GB import.num-threads: 64 memQuotaQuery := uint64(1073741824) if benchType == "ch" || benchType == "tpch" || benchType == "tpcds" || benchType == "job" || benchType == "apdogfood" { memQuotaQuery = 84825604096 } configMap := make(map[string]interface{}) err := yaml.Unmarshal([]byte(config), &configMap) if err != nil { return "", err } if componentType == "tidb" { // provide default mem-quota-query if it is not specified if _, ok := configMap["mem-quota-query"]; !ok { configMap["mem-quota-query"] = memQuotaQuery } // provide new collation if it's not specified if _, ok := configMap["new_collations_enabled_on_first_bootstrap"]; !ok { configMap["new_collations_enabled_on_first_bootstrap"] = false } } if componentType == "tikv" { // provide default mem-quota-query if it is not specified if _, ok := configMap["storage"]; ok { map1 := configMap["storage"].(map[string]interface{}) if _, ok2 := map1["block-cache"]; ok2 { map2 := map1["block-cache"].(map[string]interface{}) if _, ok3 := map2["capacity"]; !ok3 { configMap["storage.block-cache.capacity"] = blockCache } } else { configMap["storage.block-cache.capacity"] = blockCache } } else { configMap["storage.block-cache.capacity"] = blockCache } // provide default mem-quota-query if it is not specified if _, ok := configMap["import"]; ok { map1 := configMap["import"].(map[string]interface{}) if _, ok2 := map1["num-threads"]; !ok2 { configMap["import.num-threads"] = numThreads } } else { configMap["import.num-threads"] = numThreads } } configStr, err := MapToJson(configMap) if err != nil { return "", err } return string(configStr), nil } func saveOlapResult(db *sql.DB, db2 *sql.DB, benchEnv string, email string, benchType string, benchSubType string, tidbVersion string, tidbHash string, tikvVersion string, tikvHash string, pdVersion string, pdHash string, tiflashVersion string, tiflashHash string, beginAt int64, endAt int64, seconds int64, testbedName string, idle_80_not_in_txn, idle_80_in_txn, parse_80, compile_80, execute_80, tidb_prewrite_80, tidb_kv_get_80, tidb_coprocessor_80, tidb_cpu_avg, tidb_cpu_delta_avg, plan_cache_ops, pd_tso_80, tikv_cpu_avg, tikv_cpu_delta_avg, tikv_coprocessor_80, tikv_kv_get_80, tikv_prewrite_80, apply_80, append_80, commit_80, idle_99_not_in_txn, idle_99_in_txn, parse_99, compile_99, execute_99, tidb_prewrite_99, tidb_kv_get_99, tidb_coprocessor_99, pd_tso_99, tikv_coprocessor_99, tikv_kv_get_99, tikv_prewrite_99, apply_99, append_99, commit_99, idle_avg_not_in_txn, idle_avg_in_txn, parse_avg, compile_avg, execute_avg, tidb_prewrite_avg, tidb_kv_get_avg, tidb_coprocessor_avg, pd_tso_avg, tikv_coprocessor_avg, tikv_kv_get_avg, tikv_prewrite_avg, apply_avg, append_avg, commit_avg float64, benchmarkID string) error { rows, err := db2.Query(summarySlowQuery) if err != nil { return err } defer rows.Close() var summaryLogID, count, planCount int64 var query, digest, plan, planDigest string var latency, minLatency, avgLatency, maxLatency float64 for rows.Next() { err = rows.Scan(&count, &minLatency, &avgLatency, &maxLatency) if err != nil { return err } // insert the olap_summary_logs record res, err := db.Exec(olapSummaryInsert, benchEnv, email, benchType, benchSubType, tidbVersion, tidbHash, tikvVersion, tikvHash, pdVersion, pdHash, tiflashVersion, tiflashHash, count, count/seconds*60.0, minLatency, avgLatency, maxLatency, beginAt, endAt, seconds, testbedName, idle_80_not_in_txn, idle_80_in_txn, parse_80, compile_80, execute_80, tidb_prewrite_80, tidb_kv_get_80, tidb_coprocessor_80, tidb_cpu_avg, tidb_cpu_delta_avg, plan_cache_ops, pd_tso_80, tikv_cpu_avg, tikv_cpu_delta_avg, tikv_coprocessor_80, tikv_kv_get_80, tikv_prewrite_80, apply_80, append_80, commit_80, idle_99_not_in_txn, idle_99_in_txn, parse_99, compile_99, execute_99, tidb_prewrite_99, tidb_kv_get_99, tidb_coprocessor_99, pd_tso_99, tikv_coprocessor_99, tikv_kv_get_99, tikv_prewrite_99, apply_99, append_99, commit_99, idle_avg_not_in_txn, idle_avg_in_txn, parse_avg, compile_avg, execute_avg, tidb_prewrite_avg, tidb_kv_get_avg, tidb_coprocessor_avg, pd_tso_avg, tikv_coprocessor_avg, tikv_kv_get_avg, tikv_prewrite_avg, apply_avg, append_avg, commit_avg, benchmarkID) if err != nil { return err } else { id, err := res.LastInsertId() if err != nil { return err } else { summaryLogID = id } } } rows2, err2 := db2.Query(itemSlowQuery) if err2 != nil { return err2 } defer rows2.Close() for rows2.Next() { err = rows2.Scan(&digest, &count, &planCount, &minLatency, &avgLatency, &maxLatency) if err != nil { return err } // insert the olap_summary_item_logs record _, err := db.Exec(olapItemInsert, summaryLogID, digest, count, planCount, minLatency, avgLatency, maxLatency) if err != nil { return err } } rows3, err3 := db2.Query(detailSlowQuery) if err3 != nil { return err3 } defer rows3.Close() log.Info("start saving for olap detail logs") for rows3.Next() { err = rows3.Scan(&query, &digest, &plan, &planDigest, &latency) //log.Info(digest) if err != nil { return err } // insert the olap_detail_logs record _, err := db.Exec(olapDetailInsert, summaryLogID, query, digest, plan, planDigest, latency) if err != nil { return err } } return nil } func tailLog(benchType string, workload resource.WorkloadNode, ctx context.Context) { os.Setenv("ENDLESS_CMD_STDOUT", "stdout") tailCmd := workloadnode.CommandContext(ctx, fmt.Sprintf("tail -100 %s.output", benchType)) // tailCmd.Stdout = os.Stdout tailCmd.Run(workload) os.Setenv("ENDLESS_CMD_STDOUT", "") } func runBackendCommandsCloud(password, dbHost, dbPort, dbName, commands string, delay time.Duration, workload resource.WorkloadNode, ctx context.Context) { if len(commands) > 0 { os.Setenv("ENDLESS_CMD_STDOUT", "stdout") log.Info(fmt.Sprintf("commands delay begins: %v", delay)) time.Sleep(delay) log.Info("commands begins") s1 := fmt.Sprintf(mysqlCmdTplCloud, password, dbHost, dbPort, dbName, commands) mysqlCmd := workloadnode.CommandContext(ctx, s1) mysqlCmd.Run(workload) log.Info("commmands finished.") os.Setenv("ENDLESS_CMD_STDOUT", "") } } func runBackendCommands(dbHost, dbPort, dbName, commands string, delay time.Duration, workload resource.WorkloadNode, ctx context.Context) { if len(commands) > 0 { os.Setenv("ENDLESS_CMD_STDOUT", "stdout") log.Info(fmt.Sprintf("commands delay begins: %v", delay)) time.Sleep(delay) log.Info("commands begins") s1 := fmt.Sprintf(mysqlCmdTpl, dbHost, dbPort, dbName, commands) mysqlCmd := workloadnode.CommandContext(ctx, s1) mysqlCmd.Run(workload) log.Info("commmands finished.") os.Setenv("ENDLESS_CMD_STDOUT", "") } } func getBenchCmdCloud(benchType, benchSubType, dbHost, dbPort, dbName, thread, apThread string, duration time.Duration, warehouses int, sysbenchOptions, ycsbOptions string, queries string, tiflashReplicas int, jdbc, password string) string { var cmd string switch { case benchType == "tpcc": cmd = fmt.Sprintf(tpccCmdTplCloud, dbHost, dbPort, warehouses, dbName, thread, duration.String(), password) case benchType == "benchmarksql": cmd = fmt.Sprintf(benchmarksqlCmdTplCloud, dbHost, dbPort, dbName, warehouses, thread, int(duration.Minutes()), jdbc, password) case benchType == "ch": cmd = fmt.Sprintf(chCmdTplCloud, dbHost, dbPort, warehouses, dbName, thread, apThread, queries, duration.String(), password) case benchType == "ycsb": cmd = fmt.Sprintf(ycsbCmdTplCloud, benchSubType, dbHost, dbPort, thread, ycsbOptions, password) case benchType == "sysbench": cmd = fmt.Sprintf(sysbenchCmdTplCloud, benchSubType, dbHost, dbPort, dbName, thread, int(duration.Seconds()), sysbenchOptions, password) case benchType == "tpch" && tiflashReplicas == 0: cmd = fmt.Sprintf(tpchCmdTplCloud, dbHost, dbPort, dbName, queries, duration.String(), password) case benchType == "tpch" && tiflashReplicas > 0: cmd = fmt.Sprintf(tpchCmdTpl2Cloud, dbHost, dbPort, dbName, queries, duration.String(), password) case benchType == "tpcds": cmd = "cd tpcds50; " + fmt.Sprintf(rawSQLCmdTplCloud, password, dbHost, dbPort, dbName, queries, duration.String(), benchType) case benchType == "job": cmd = "cd job; " + fmt.Sprintf(rawSQLCmdTplCloud, password, dbHost, dbPort, dbName, queries, duration.String(), benchType) case benchType == "bank": cmd = fmt.Sprintf(bankCmdTplCloud, password, dbHost, dbPort, dbName, thread, int(duration.Seconds())) case benchType == "shenma": cmd = fmt.Sprintf(shenmaCmdTplCloud, password, dbHost, dbPort, thread) case benchType == "corebanking" || benchType == "hzbank" || benchType == "booking": cmd = fmt.Sprintf(corebankingCmdTplCloud, dbHost, dbPort, dbName, int(duration.Seconds()), thread, jdbc, benchType, password) } return cmd } func getBenchCmd(benchType, benchSubType, dbHost, dbPort, dbName, thread, apThread string, duration time.Duration, warehouses int, sysbenchOptions, ycsbOptions string, queries string, tiflashReplicas int, jdbc string) string { var cmd string switch { case benchType == "taobench": cmd = fmt.Sprintf(taoCmdTpl, dbHost, thread, benchSubType) case benchType == "tpcc": cmd = fmt.Sprintf(tpccCmdTpl, dbHost, dbPort, warehouses, dbName, thread, duration.String()) case benchType == "benchmarksql": cmd = fmt.Sprintf(benchmarksqlCmdTpl, dbHost, dbPort, dbName, warehouses, thread, int(duration.Minutes()), jdbc) case benchType == "ch": cmd = fmt.Sprintf(chCmdTpl, dbHost, dbPort, warehouses, dbName, thread, apThread, queries, duration.String()) case benchType == "ycsb": cmd = fmt.Sprintf(ycsbCmdTpl, benchSubType, dbHost, dbPort, thread, ycsbOptions) case benchType == "sysbench": cmd = fmt.Sprintf(sysbenchCmdTpl, benchSubType, dbHost, dbPort, dbName, thread, int(duration.Seconds()), sysbenchOptions) case benchType == "tpch" && tiflashReplicas == 0: cmd = fmt.Sprintf(tpchCmdTpl, dbHost, dbPort, dbName, queries, duration.String()) case benchType == "tpch" && tiflashReplicas > 0: cmd = fmt.Sprintf(tpchCmdTpl2, dbHost, dbPort, dbName, queries, duration.String()) case benchType == "tpcds": cmd = "cd tpcds50; " + fmt.Sprintf(rawSQLCmdTpl, dbHost, dbPort, dbName, queries, duration.String(), benchType) case benchType == "job": cmd = "cd job; " + fmt.Sprintf(rawSQLCmdTpl, dbHost, dbPort, dbName, queries, duration.String(), benchType) case benchType == "bank": cmd = fmt.Sprintf(bankCmdTpl, dbHost, dbPort, dbName, thread, int(duration.Seconds())) case benchType == "shenma": cmd = fmt.Sprintf(shenmaCmdTpl, dbHost, dbPort, thread) case benchType == "corebanking" || benchType == "hzbank" || benchType == "booking": cmd = fmt.Sprintf(corebankingCmdTpl, dbHost, dbPort, dbName, int(duration.Seconds()), thread, jdbc, benchType) case benchType == "apdogfood": cmd = fmt.Sprintf("cd apdogfood; sh ./run.sh %s %s", dbHost, dbPort) } return cmd } func getTiFlashTableList(benchType string, tiflashReplicas int) string { tableList := "" if benchType == "tpch" { tableList = fmt.Sprintf("alter table supplier set tiflash replica %[1]v; alter table region set tiflash replica %[1]v; alter table partsupp set tiflash replica %[1]v; alter table part set tiflash replica %[1]v; alter table nation set tiflash replica %[1]v; alter table orders set tiflash replica %[1]v; alter table lineitem set tiflash replica %[1]v; alter table customer set tiflash replica %[1]v;", tiflashReplicas) } else if benchType == "tpcds" { tableList = fmt.Sprintf("alter table call_center set tiflash replica %[1]v; alter table catalog_page set tiflash replica %[1]v; alter table catalog_returns set tiflash replica %[1]v; alter table catalog_sales set tiflash replica %[1]v; alter table customer set tiflash replica %[1]v; alter table customer_address set tiflash replica %[1]v; alter table customer_demographics set tiflash replica %[1]v; alter table date_dim set tiflash replica %[1]v; alter table dbgen_version set tiflash replica %[1]v; alter table household_demographics set tiflash replica %[1]v; alter table income_band set tiflash replica %[1]v; alter table inventory set tiflash replica %[1]v; alter table item set tiflash replica %[1]v; alter table promotion set tiflash replica %[1]v; alter table reason set tiflash replica %[1]v; alter table ship_mode set tiflash replica %[1]v; alter table store set tiflash replica %[1]v; alter table store_returns set tiflash replica %[1]v; alter table store_sales set tiflash replica %[1]v; alter table time_dim set tiflash replica %[1]v; alter table warehouse set tiflash replica %[1]v; alter table web_page set tiflash replica %[1]v; alter table web_returns set tiflash replica %[1]v; alter table web_sales set tiflash replica %[1]v; alter table web_site set tiflash replica %[1]v;", tiflashReplicas) } else if benchType == "ch" { tableList = fmt.Sprintf("alter table customer set tiflash replica %[1]v;alter table district set tiflash replica %[1]v;alter table history set tiflash replica %[1]v;alter table item set tiflash replica %[1]v;alter table new_order set tiflash replica %[1]v;alter table order_line set tiflash replica %[1]v;alter table orders set tiflash replica %[1]v;alter table stock set tiflash replica %[1]v;alter table warehouse set tiflash replica %[1]v;alter table nation set tiflash replica %[1]v;alter table region set tiflash replica %[1]v;alter table supplier set tiflash replica %[1]v;", tiflashReplicas) } else if benchType == "apdogfood" { tableList = fmt.Sprintf("alter table `ads_demo`.`ads` set tiflash replica %[1]v;alter table `ads_demo`.`third_party_data` set tiflash replica %[1]v;alter table `ads_demo`.`device` set tiflash replica %[1]v;alter table `ads_demo`.`click_record` set tiflash replica %[1]v;alter table `ads_demo`.`transform_callback` set tiflash replica %[1]v;alter table `discourse`.`user_visits` set tiflash replica %[1]v;alter table `pingcap_account`.`track_events` set tiflash replica %[1]v;alter table `pingcap_unversity`.`t1` set tiflash replica %[1]v;alter table `software_distribution`.`docker` set tiflash replica %[1]v;alter table `software_distribution`.`tiup` set tiflash replica %[1]v;alter table `software_distribution`.`download` set tiflash replica %[1]v;alter table `sync_ci_data`.`ci_data` set tiflash replica %[1]v;alter table `warehouse`.`db-engine-trends` set tiflash replica %[1]v;alter table `warehouse`.`human-resource-distribution` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_hardware_memory` set tiflash replica %[1]v;alter table `warehouse`.`modb-dbrank` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_hardware_disk` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_sql_usage` set tiflash replica %[1]v;alter table `warehouse`.`pingcap-office-geo` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_hardware_basic` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_host_extra` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_sql_usage_total` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_feature_usage` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_copr_cache_usage` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_windowed_stats` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_hardware_cpu` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_txn` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_basic_info` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_clustered_index` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_slow_query_bucket` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_clustered_index_table` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_instance` set tiflash replica %[1]v;alter table `warehouse`.`telemetry_tidb_tiflash` set tiflash replica %[1]v;", tiflashReplicas) } return tableList } func getAnalyzeList(benchType string) string { tableList := "" if benchType == "taobench" { tableList = "set tidb_build_stats_concurrency=16; set tidb_distsql_scan_concurrency=16; set tidb_index_serial_scan_concurrency=16; analyze table objects; analyze table edges;" } else if benchType == "tpcc" { tableList = "set tidb_build_stats_concurrency=16; set tidb_distsql_scan_concurrency=16; set tidb_index_serial_scan_concurrency=16; analyze table customer; analyze table district; analyze table history; analyze table item; analyze table new_order; analyze table order_line; analyze table orders; analyze table stock; analyze table warehouse;" } else if benchType == "benchmarksql" { tableList = "set tidb_build_stats_concurrency=16; set tidb_distsql_scan_concurrency=16; set tidb_index_serial_scan_concurrency=16; analyze table bmsql_config;analyze table bmsql_warehouse;analyze table bmsql_district;analyze table bmsql_customer;analyze table bmsql_history;analyze table bmsql_new_order;analyze table bmsql_oorder;analyze table bmsql_order_line;analyze table bmsql_item;analyze table bmsql_stock;" } else if benchType == "ch" { tableList = "set tidb_build_stats_concurrency=16; set tidb_distsql_scan_concurrency=16; set tidb_index_serial_scan_concurrency=16; analyze table customer; analyze table district; analyze table history; analyze table item; analyze table new_order; analyze table order_line; analyze table orders; analyze table stock; analyze table warehouse; analyze table nation;analyze table region; analyze table supplier" } else if benchType == "tpch" { tableList = "set tidb_build_stats_concurrency=32; set tidb_distsql_scan_concurrency=30; set tidb_index_serial_scan_concurrency=32; analyze table lineitem; analyze table orders; analyze table part; analyze table partsupp; analyze table supplier; analyze table customer; analyze table region; analyze table nation;" } else if benchType == "tpcds" { tableList = "set tidb_build_stats_concurrency=32; set tidb_distsql_scan_concurrency=30; set tidb_index_serial_scan_concurrency=32; analyze table call_center; analyze table catalog_page; analyze table catalog_returns; analyze table catalog_sales; analyze table customer; analyze table customer_address; analyze table customer_demographics; analyze table date_dim; analyze table dbgen_version; analyze table household_demographics; analyze table income_band; analyze table inventory; analyze table item; analyze table promotion; analyze table reason; analyze table ship_mode; analyze table store; analyze table store_returns; analyze table store_sales; analyze table time_dim; analyze table warehouse; analyze table web_page; analyze table web_returns; analyze table web_sales; analyze table web_site;" } else if benchType == "job" { tableList = "analyze table aka_name; analyze table aka_title; analyze table cast_info; analyze table char_name; analyze table comp_cast_type; analyze table company_name; analyze table company_type; analyze table complete_cast; analyze table info_type; analyze table keyword; analyze table kind_type; analyze table link_type; analyze table movie_companies; analyze table movie_info; analyze table movie_info_idx; analyze table movie_keyword; analyze table movie_link; analyze table name; analyze table person_info; analyze table role_type; analyze table title;" } else if benchType == "bank" { tableList = "analyze table account_detail;" } else if benchType == "booking" { tableList = "analyze table accommodations_items;" } else if benchType == "apdogfood" { tableList = "set tidb_build_stats_concurrency=32; set tidb_distsql_scan_concurrency=30; set tidb_index_serial_scan_concurrency=32; analyze table `discourse`.`tag_groups`;analyze table `discourse`.`badge_types`;analyze table `discourse`.`topic_embeds`;analyze table `discourse`.`user_associated_accounts`;analyze table `discourse`.`group_custom_fields`;analyze table `discourse`.`users`;analyze table `discourse`.`drafts`;analyze table `discourse`.`google_user_infos`;analyze table `discourse`.`user_profiles`;analyze table `discourse`.`site_settings`;analyze table `discourse`.`ar_internal_metadata`;analyze table `discourse`.`anonymous_users`;analyze table `discourse`.`category_featured_topics`;analyze table `discourse`.`post_revisions`;analyze table `discourse`.`user_archived_messages`;analyze table `discourse`.`user_open_ids`;analyze table `discourse`.`shared_drafts`;analyze table `discourse`.`reviewable_claimed_topics`;analyze table `discourse`.`email_change_requests`;analyze table `discourse`.`user_api_keys`;analyze table `discourse`.`schema_migrations`;analyze table `discourse`.`category_custom_fields`;analyze table `discourse`.`custom_emojis`;analyze table `discourse`.`post_replies`;analyze table `discourse`.`invites`;analyze table `discourse`.`developers`;analyze table `discourse`.`category_tag_groups`;analyze table `discourse`.`application_requests`;analyze table `discourse`.`categories`;analyze table `discourse`.`skipped_email_logs`;analyze table `discourse`.`category_users`;analyze table `discourse`.`categories_web_hooks`;analyze table `discourse`.`web_hook_event_types_hooks`;analyze table `discourse`.`message_bus`;analyze table `discourse`.`user_avatars`;analyze table `discourse`.`oauth2_user_infos`;analyze table `discourse`.`topic_tags`;analyze table `discourse`.`quoted_posts`;analyze table `discourse`.`user_custom_fields`;analyze table `discourse`.`tx1`;analyze table `discourse`.`scheduler_stats`;analyze table `discourse`.`reviewables`;analyze table `discourse`.`user_badges`;analyze table `discourse`.`poll_options`;analyze table `discourse`.`badge_groupings`;analyze table `discourse`.`category_tag_stats`;analyze table `discourse`.`tags_web_hooks`;analyze table `discourse`.`tx`;analyze table `discourse`.`permalinks`;analyze table `discourse`.`user_options`;analyze table `discourse`.`web_crawler_requests`;analyze table `discourse`.`topic_allowed_groups`;analyze table `discourse`.`reviewable_scores`;analyze table `discourse`.`tag_users`;analyze table `discourse`.`tag_group_permissions`;analyze table `discourse`.`groups_web_hooks`;analyze table `discourse`.`category_groups`;analyze table `discourse`.`group_users`;analyze table `discourse`.`single_sign_on_records`;analyze table `discourse`.`user_histories`;analyze table `discourse`.`user_second_factors`;analyze table `discourse`.`api_keys`;analyze table `discourse`.`post_stats`;analyze table `discourse`.`color_schemes`;analyze table `discourse`.`poll_votes`;analyze table `discourse`.`post_search_data`;analyze table `discourse`.`email_logs`;analyze table `discourse`.`ignored_users`;analyze table `discourse`.`incoming_referers`;analyze table `discourse`.`tags`;analyze table `discourse`.`screened_emails`;analyze table `discourse`.`tag_group_memberships`;analyze table `discourse`.`group_archived_messages`;analyze table `discourse`.`uploads`;analyze table `discourse`.`post_custom_fields`;analyze table `discourse`.`plugin_store_rows`;analyze table `discourse`.`topic_search_data`;analyze table `discourse`.`post_uploads`;analyze table `discourse`.`topic_timers`;analyze table `discourse`.`topic_link_clicks`;analyze table `discourse`.`directory_items`;analyze table `discourse`.`given_daily_likes`;analyze table `discourse`.`notifications`;analyze table `discourse`.`theme_translation_overrides`;analyze table `discourse`.`optimized_images`;analyze table `discourse`.`user_uploads`;analyze table `discourse`.`post_action_types`;analyze table `discourse`.`instagram_user_infos`;analyze table `discourse`.`stylesheet_cache`;analyze table `discourse`.`category_tags`;analyze table `discourse`.`search_logs`;analyze table `discourse`.`user_fields`;analyze table `discourse`.`post_actions`;analyze table `discourse`.`user_field_options`;analyze table `discourse`.`incoming_links`;analyze table `discourse`.`embeddable_hosts`;analyze table `discourse`.`theme_fields`;analyze table `discourse`.`schema_migration_details`;analyze table `discourse`.`user_emails`;analyze table `discourse`.`topic_invites`;analyze table `discourse`.`posts`;analyze table `discourse`.`top_topics`;analyze table `discourse`.`one_row_table`;analyze table `discourse`.`screened_ip_addresses`;analyze table `discourse`.`reviewable_histories`;analyze table `discourse`.`post_reply_keys`;analyze table `discourse`.`user_auth_tokens`;analyze table `discourse`.`post_timings`;analyze table `discourse`.`topic_links`;analyze table `discourse`.`topic_custom_fields`;analyze table `discourse`.`child_themes`;analyze table `discourse`.`badges`;analyze table `discourse`.`onceoff_logs`;analyze table `discourse`.`topic_views`;analyze table `discourse`.`color_scheme_colors`;analyze table `discourse`.`user_visits`;analyze table `discourse`.`translation_overrides`;analyze table `discourse`.`topic_users`;analyze table `discourse`.`email_tokens`;analyze table `discourse`.`invited_groups`;analyze table `discourse`.`user_stats`;analyze table `discourse`.`draft_sequences`;analyze table `discourse`.`javascript_caches`;analyze table `discourse`.`watched_words`;analyze table `discourse`.`polls`;analyze table `discourse`.`unsubscribe_keys`;analyze table `discourse`.`tag_search_data`;analyze table `discourse`.`category_search_data`;analyze table `discourse`.`user_warnings`;analyze table `discourse`.`muted_users`;analyze table `discourse`.`group_requests`;analyze table `discourse`.`user_exports`;analyze table `discourse`.`remote_themes`;analyze table `discourse`.`user_search_data`;analyze table `discourse`.`groups`;analyze table `discourse`.`web_hook_events`;analyze table `discourse`.`user_auth_token_logs`;analyze table `discourse`.`theme_settings`;analyze table `discourse`.`topics`;analyze table `discourse`.`themes`;analyze table `discourse`.`user_actions`;analyze table `discourse`.`github_user_infos`;analyze table `discourse`.`screened_urls`;analyze table `discourse`.`topic_allowed_users`;analyze table `discourse`.`group_histories`;analyze table `discourse`.`incoming_emails`;analyze table `discourse`.`web_hook_event_types`;analyze table `discourse`.`user_profile_views`;analyze table `discourse`.`incoming_domains`;analyze table `discourse`.`post_details`;analyze table `discourse`.`web_hooks`;analyze table `discourse`.`push_subscriptions`;analyze table `discourse`.`group_mentions`;analyze table `warehouse`.`db-engine-trends`;analyze table `warehouse`.`qa_quality_infra_tmp`;analyze table `warehouse`.`hr_employees`;analyze table `warehouse`.`baidu-index`;analyze table `warehouse`.`pingcap-salesforce-order`;analyze table `warehouse`.`questionnaires`;analyze table `warehouse`.`human-resource-distribution`;analyze table `warehouse`.`questionnaire_answers`;analyze table `warehouse`.`pingcap-salesforce-lead`;analyze table `warehouse`.`qa_oncall_diff_tmp`;analyze table `warehouse`.`pingcap-crm-order`;analyze table `warehouse`.`qa_converage_tmp`;analyze table `warehouse`.`hr_employee_departments`;analyze table `warehouse`.`pingcap-crm-order-new`;analyze table `warehouse`.`pingcap-crm-payment`;analyze table `warehouse`.`pingcap-crm-business-opportunity`;analyze table `warehouse`.`telemetry_tidb_hardware_memory`;analyze table `warehouse`.`modb-dbrank`;analyze table `warehouse`.`qa_quality_tmp`;analyze table `warehouse`.`telemetry_tidb_hardware_disk`;analyze table `warehouse`.`telemetry_tidb_sql_usage`;analyze table `warehouse`.`pingcap-office-geo`;analyze table `warehouse`.`telemetry_tidb_hardware_basic`;analyze table `warehouse`.`telemetry_tidb_host_extra`;analyze table `warehouse`.`pingcap-customers`;analyze table `warehouse`.`questionnaire_messages`;analyze table `warehouse`.`telemetry_tidb_sql_usage_total`;analyze table `warehouse`.`telemetry_tidb_feature_usage`;analyze table `warehouse`.`telemetry_tidb_copr_cache_usage`;analyze table `warehouse`.`telemetry_tidb_windowed_stats`;analyze table `warehouse`.`telemetry_tidb_hardware_cpu`;analyze table `warehouse`.`cdn_download_log_v1`;analyze table `warehouse`.`telemetry_tidb_txn`;analyze table `warehouse`.`telemetry_tidb_basic_info`;analyze table `warehouse`.`telemetry_tidb_clustered_index`;analyze table `warehouse`.`pingcap-salesforce-opportunity`;analyze table `warehouse`.`telemetry_tidb_slow_query_bucket`;analyze table `warehouse`.`hr_departments`;analyze table `warehouse`.`cdn_download_logs`;analyze table `warehouse`.`telemetry_tidb_clustered_index_table`;analyze table `warehouse`.`telemetry_tidb_instance`;analyze table `warehouse`.`telemetry_tidb_tiflash`;" } return tableList } func getWarmupCmd(benchType, benchSubType, dbHost, dbPort, dbName, thread, apThread string, duration time.Duration, warehouses int, sysbenchOptions, ycsbOptions, jdbc string) string { cmd := "" switch benchType { case "tpcc": cmd = fmt.Sprintf(tpccCmdTpl, dbHost, dbPort, warehouses, dbName, thread, duration.String()) case "benchmarksql": cmd = fmt.Sprintf(benchmarksqlCmdTpl, dbHost, dbPort, dbName, warehouses, thread, int(duration.Minutes()), jdbc) // case "ch": // cmd = "cd ch; " + fmt.Sprintf("mysql --host %s -P%s -D %s < all.sql > /dev/null ", dbHost, dbPort, dbName) + " && sleep 120" // cmd = fmt.Sprintf(chCmdTpl, dbHost, dbPort, warehouses, dbName, thread, apThread, duration.String()) case "sysbench": cmd = fmt.Sprintf(sysbenchCmdTpl, benchSubType, dbHost, dbPort, dbName, thread, int(duration.Seconds()), sysbenchOptions) case "bank": cmd = fmt.Sprintf(bankCmdTpl, dbHost, dbPort, dbName, thread, int(duration.Seconds())) case "tpch": cmd = "cd tpch; " + fmt.Sprintf("mysql --host %s -P%s -D %s < all.sql > /dev/null ", dbHost, dbPort, dbName) + " && sleep 120" case "tpcds": cmd = "cd tpcds50; " + fmt.Sprintf("mysql --host %s -P%s -D %s < all.sql > /dev/null ", dbHost, dbPort, dbName) + " && sleep 120" case "job": cmd = "cd job; " + fmt.Sprintf("mysql --host %s -P%s -D %s < all.sql > /dev/null ", dbHost, dbPort, dbName) + " && sleep 120" } return cmd } func defaultDataSetCloud(benchType string) (string, string) { switch benchType { case "sysbench": return "benchmark/sysbench-32-1000w-auto", "test" case "ycsb": return "benchmark/ycsb-100m-v5.0.0", "test" // for 4.x s3://benchmark/ycsb-100m-release4.0, for 3.x s3://benchmark/ycsb-100m case "tpcc": return "benchmark/tpcc-1000-new", "tpcc" case "benchmarksql": return "benchmark/benchmarksql-tpcc-1k", "benchmarksql" case "ch": return "benchmark/ch-1k-v5", "tpcc" case "bank": return "bank/bank-50m", "bank" case "shenma": return "bank/shenma-full-with-stats", "test" case "corebanking": return "benchmark/corebanking", "hzbank" case "hzbank": return "benchmark/hzbank5000-20220207-varchar-length-extended", "heronsdb3" case "booking": return "benchmark/booking-v6.0", "test" case "tpch": return "benchmark/tpch50", "test" case "tpcds": return "benchmark/tpcds50", "tpcds50" case "job": return "benchmark/imdb", "imdb" } return "", "" } func defaultDataSet(benchType string) (string, string) { switch benchType { case "sysbench": return "s3://benchmark/sysbench-32-1000w-auto", "test" case "ycsb": return "s3://benchmark/ycsb-100m-v5.0.0", "test" // for 4.x s3://benchmark/ycsb-100m-release4.0, for 3.x s3://benchmark/ycsb-100m case "tpcc": return "s3://benchmark/tpcc-1000-new", "tpcc" case "benchmarksql": return "s3://benchmark/benchmarksql-tpcc-1k", "benchmarksql" case "ch": return "s3://benchmark/ch-1k-v5", "tpcc" case "bank": return "s3://bank/bank-50m", "bank" case "shenma": return "s3://bank/shenma-full-with-stats/", "test" case "corebanking": return "s3://benchmark/corebanking", "hzbank" case "hzbank": return "s3://benchmark/hzbank5000-20220207-varchar-length-extended", "heronsdb3" case "booking": return "s3://benchmark/booking-v6.0", "test" case "tpch": return "s3://benchmark/tpch50", "test" case "tpcds": return "s3://benchmark/tpcds50", "tpcds50" case "job": return "s3://benchmark/imdb/", "imdb" case "apdogfood": return "s3://nfs/tiflash/cglnew", "test" case "taobench": return "s3://bank/tao_165m/", "benchmark" } return "", "" } func defaultQueries(benchType string) string { switch benchType { case "tpch": return "q1,q2,q3,q4,q5,q6,q7,q8,q9,q10,q11,q12,q13,q14,q15,q16,q17,q18,q19,q20,q21,q22" case "ch": return "q1,q2,q3,q4,q5,q6,q7,q8,q9,q10,q11,q12,q13,q14,q15,q16,q17,q18,q19,q20,q21,q22" case "tpcds": return "query_1.sql,query_3.sql,query_6.sql,query_7.sql,query_8.sql,query_9.sql,query_10.sql,query_12.sql,query_13.sql,query_15.sql,query_16.sql,query_17.sql,query_19.sql,query_20.sql,query_21.sql,query_24_1.sql,query_24_0.sql,query_25.sql,query_26.sql,query_28.sql,query_29.sql,query_30.sql,query_31.sql,query_32.sql,query_33.sql,query_34.sql,query_35.sql,query_37.sql,query_40.sql,query_41.sql,query_42.sql,query_43.sql,query_44.sql,query_45.sql,query_46.sql,query_47.sql,query_48.sql,query_49.sql,query_50.sql,query_51.sql,query_52.sql,query_53.sql,query_54.sql,query_55.sql,query_56.sql,query_57.sql,query_58.sql,query_60.sql,query_61.sql,query_62.sql,query_63.sql,query_64.sql,query_65.sql,query_66.sql,query_68.sql,query_69.sql,query_71.sql,query_72.sql,query_73.sql,query_74.sql,query_75.sql,query_76.sql,query_79.sql,query_81.sql,query_82.sql,query_83.sql,query_84.sql,query_87.sql,query_88.sql,query_89.sql,query_90.sql,query_91.sql,query_92.sql,query_93.sql,query_94.sql,query_95.sql,query_96.sql,query_97.sql,query_98.sql,query_99.sql" case "job": return "10a.sql,10b.sql,10c.sql,11a.sql,11b.sql,11c.sql,11d.sql,12a.sql,12b.sql,12c.sql,13a.sql,13b.sql,13c.sql,13d.sql,14a.sql,14b.sql,14c.sql,15a.sql,15b.sql,15c.sql,15d.sql,16a.sql,16b.sql,16c.sql,16d.sql,17a.sql,17b.sql,17c.sql,17d.sql,17e.sql,17f.sql,18a.sql,18b.sql,18c.sql,19a.sql,19b.sql,19c.sql,19d.sql,1a.sql,1b.sql,1c.sql,1d.sql,20a.sql,20b.sql,20c.sql,21a.sql,21b.sql,21c.sql,22a.sql,22b.sql,22c.sql,22d.sql,23a.sql,23b.sql,23c.sql,24a.sql,24b.sql,25a.sql,25b.sql,25c.sql,26a.sql,26b.sql,26c.sql,27a.sql,27b.sql,27c.sql,28a.sql,28b.sql,28c.sql,29a.sql,29b.sql,29c.sql,2a.sql,2b.sql,2c.sql,2d.sql,30a.sql,30b.sql,30c.sql,31a.sql,31b.sql,31c.sql,32a.sql,32b.sql,33a.sql,33b.sql,33c.sql,3a.sql,3b.sql,3c.sql,4a.sql,4b.sql,4c.sql,5a.sql,5b.sql,5c.sql,6a.sql,6b.sql,6c.sql,6d.sql,6e.sql,6f.sql,7a.sql,7b.sql,7c.sql,8a.sql,8b.sql,8c.sql,8d.sql,9a.sql,9b.sql,9c.sql,9d.sql" } return "" } func getBenchResult(promURL *url.URL, seconds int64, endTime time.Time) (float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64) { promAPI, err := newPrometheusAPI(promURL.String()) if err != nil { return 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 } return getBenchResultWithPromAPI(promAPI, seconds, endTime) } func newPrometheusAPI(prometheusAddr string) (v1.API, error) { client, err := api.NewClient(api.Config{ Address: prometheusAddr, }) if err != nil { return nil, err } promAPI := v1.NewAPI(client) return promAPI, nil } func getBenchResultWithPromAPI(promAPI v1.API, seconds int64, endTime time.Time) (float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64, float64) { avgQps, _ := getMetric(promAPI, fmt.Sprintf(`sum(rate(tidb_server_query_total[%ds]))`, seconds), endTime) avgTps, _ := getMetric(promAPI, fmt.Sprintf(`sum(rate(tidb_session_transaction_duration_seconds_count[%ds]))`, seconds), endTime) avgP90, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.90, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) avgP99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) avgP999, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) //avg latency, max latency avgLatency, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tidb_server_handle_query_duration_seconds_sum{sql_type!="internal"}[%ds])) / sum(rate(tidb_server_handle_query_duration_seconds_count{sql_type!="internal"}[%ds])))`, seconds, seconds), endTime) maxLatency, _ := getMetric(promAPI, fmt.Sprintf(`1000*max_over_time(histogram_quantile(1, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) //min minQps, _ := getMetric(promAPI, fmt.Sprintf(`min_over_time(sum(rate(tidb_server_query_total[1m])) [%ds:])`, seconds), endTime) minTps, _ := getMetric(promAPI, fmt.Sprintf(`min_over_time(sum(rate(tidb_session_transaction_duration_seconds_count[1m])) [%ds:])`, seconds), endTime) minP90, _ := getMetric(promAPI, fmt.Sprintf(`1000*min_over_time(histogram_quantile(0.90, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) minP99, _ := getMetric(promAPI, fmt.Sprintf(`1000*min_over_time(histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) minP999, _ := getMetric(promAPI, fmt.Sprintf(`1000*min_over_time(histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) //max maxQps, _ := getMetric(promAPI, fmt.Sprintf(`max_over_time(sum(rate(tidb_server_query_total[1m])) [%ds:])`, seconds), endTime) maxTps, _ := getMetric(promAPI, fmt.Sprintf(`max_over_time(sum(rate(tidb_session_transaction_duration_seconds_count[1m])) [%ds:])`, seconds), endTime) maxP90, _ := getMetric(promAPI, fmt.Sprintf(`1000*max_over_time(histogram_quantile(0.90, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) maxP99, _ := getMetric(promAPI, fmt.Sprintf(`1000*max_over_time(histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) maxP999, _ := getMetric(promAPI, fmt.Sprintf(`1000*max_over_time(histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) idle_80_not_in_txn, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='0'}[1m])) by (le))[%ds:])`, seconds), endTime) idle_80_in_txn, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='1'}[1m])) by (le))[%ds:])`, seconds), endTime) parse_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tidb_session_parse_duration_seconds_bucket{sql_type="general"}[1m])) by (le, sql_type))[%ds:])`, seconds), endTime) compile_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tidb_session_compile_duration_seconds_bucket{sql_type="general"}[1m])) by (le))[%ds:])`, seconds), endTime) execute_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tidb_session_execute_duration_seconds_bucket{sql_type="general"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_prewrite_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Prewrite"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_kv_get_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Get"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_coprocessor_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Cop"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_cpu_avg, _ := getMetric(promAPI, fmt.Sprintf(`avg(rate(process_cpu_seconds_total{job="tidb"}[%ds]))`, seconds), endTime) tidb_cpu_delta_avg, _ := getMetric(promAPI, fmt.Sprintf(`avg_over_time((max(rate(process_cpu_seconds_total{job="tidb"}[1m])) - min(rate(process_cpu_seconds_total{job="tidb"}[1m])))[%ds:])`, seconds), endTime) plan_cache_ops, _ := getMetric(promAPI, fmt.Sprintf(`avg_over_time(sum(rate(tidb_server_plan_cache_total[1m]))[%ds:])`, seconds), endTime) pd_tso_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type="wait"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_cpu_avg, _ := getMetric(promAPI, fmt.Sprintf(`avg(rate(process_cpu_seconds_total{job=~".*tikv"}[%ds]))`, seconds), endTime) tikv_cpu_delta_avg, _ := getMetric(promAPI, fmt.Sprintf(`avg_over_time((max(rate(process_cpu_seconds_total{job=~".*tikv"}[1m])) - min(rate(process_cpu_seconds_total{job=~".*tikv"}[1m])))[%ds:])`, seconds), endTime) tikv_coprocessor_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="coprocessor"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_kv_get_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="kv_get"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_prewrite_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="kv_prewrite"}[1m])) by (le))[%ds:])`, seconds), endTime) apply_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) append_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) commit_80, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) idle_99_not_in_txn, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='0'}[1m])) by (le))[%ds:])`, seconds), endTime) idle_99_in_txn, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='1'}[1m])) by (le))[%ds:])`, seconds), endTime) parse_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{sql_type="general"}[1m])) by (le, sql_type))[%ds:])`, seconds), endTime) compile_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{sql_type="general"}[1m])) by (le))[%ds:])`, seconds), endTime) execute_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tidb_session_execute_duration_seconds_bucket{sql_type="general"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_prewrite_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Prewrite"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_kv_get_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Get"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_coprocessor_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Cop"}[1m])) by (le))[%ds:])`, seconds), endTime) pd_tso_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type="wait"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_coprocessor_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="coprocessor"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_kv_get_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="kv_get"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_prewrite_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="kv_prewrite"}[1m])) by (le))[%ds:])`, seconds), endTime) apply_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) append_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) commit_99, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) p80_latency, _ := getMetric(promAPI, fmt.Sprintf(`1000*avg_over_time(histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) qps_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(sum(rate(tidb_server_query_total[1m])) [%ds:])`, seconds), endTime) p80_latency_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) p99_latency_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) p999_latency_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) idle_80_not_in_txn_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='0'}[1m])) by (le))[%ds:])`, seconds), endTime) idle_80_in_txn_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='1'}[1m])) by (le))[%ds:])`, seconds), endTime) parse_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tidb_session_parse_duration_seconds_bucket{sql_type="general"}[1m])) by (le, sql_type))[%ds:])`, seconds), endTime) compile_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tidb_session_compile_duration_seconds_bucket{sql_type="general"}[1m])) by (le))[%ds:])`, seconds), endTime) execute_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tidb_session_execute_duration_seconds_bucket{sql_type="general"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_prewrite_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Prewrite"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_kv_get_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Get"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_coprocessor_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Cop"}[1m])) by (le))[%ds:])`, seconds), endTime) pd_tso_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type="wait"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_coprocessor_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="coprocessor"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_kv_get_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="kv_get"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_prewrite_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="kv_prewrite"}[1m])) by (le))[%ds:])`, seconds), endTime) apply_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) append_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) commit_80_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.80, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) idle_99_not_in_txn_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='0'}[1m])) by (le))[%ds:])`, seconds), endTime) idle_99_in_txn_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{in_txn='1'}[1m])) by (le))[%ds:])`, seconds), endTime) parse_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{sql_type="general"}[1m])) by (le, sql_type))[%ds:])`, seconds), endTime) compile_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{sql_type="general"}[1m])) by (le))[%ds:])`, seconds), endTime) execute_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tidb_session_execute_duration_seconds_bucket{sql_type="general"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_prewrite_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Prewrite"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_kv_get_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Get"}[1m])) by (le))[%ds:])`, seconds), endTime) tidb_coprocessor_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tidb_tikvclient_request_seconds_bucket{type="Cop"}[1m])) by (le))[%ds:])`, seconds), endTime) pd_tso_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{type="wait"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_coprocessor_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="coprocessor"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_kv_get_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="kv_get"}[1m])) by (le))[%ds:])`, seconds), endTime) tikv_prewrite_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{type="kv_prewrite"}[1m])) by (le))[%ds:])`, seconds), endTime) apply_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) append_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) commit_99_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time(histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket[1m])) by (le))[%ds:])`, seconds), endTime) idle_avg_not_in_txn, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tidb_server_conn_idle_duration_seconds_sum{in_txn='0'}[%ds])) / sum(rate(tidb_server_conn_idle_duration_seconds_count{in_txn='0'}[%ds])))`, seconds, seconds), endTime) idle_avg_in_txn, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tidb_server_conn_idle_duration_seconds_sum{in_txn='1'}[%ds])) / sum(rate(tidb_server_conn_idle_duration_seconds_count{in_txn='1'}[%ds])))`, seconds, seconds), endTime) parse_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tidb_session_parse_duration_seconds_sum{sql_type="general"}[%ds])) / sum(rate(tidb_session_parse_duration_seconds_count{sql_type="general"}[%ds])))`, seconds, seconds), endTime) compile_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tidb_session_compile_duration_seconds_sum{sql_type="general"}[%ds])) / sum(rate(tidb_session_compile_duration_seconds_count{sql_type="general"}[%ds])))`, seconds, seconds), endTime) execute_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tidb_session_execute_duration_seconds_sum{sql_type="general"}[%ds])) / sum(rate(tidb_session_execute_duration_seconds_count{sql_type="general"}[%ds])))`, seconds, seconds), endTime) tidb_prewrite_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tidb_tikvclient_request_seconds_sum{type="Prewrite"}[%ds])) / sum(rate(tidb_tikvclient_request_seconds_count{type="Prewrite"}[%ds])))`, seconds, seconds), endTime) tidb_kv_get_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tidb_tikvclient_request_seconds_sum{type="Get"}[%ds])) / sum(rate(tidb_tikvclient_request_seconds_count{type="Get"}[%ds])))`, seconds, seconds), endTime) tidb_coprocessor_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tidb_tikvclient_request_seconds_sum{type="Cop"}[%ds])) / sum(rate(tidb_tikvclient_request_seconds_count{type="Cop"}[%ds])))`, seconds, seconds), endTime) pd_tso_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(pd_client_cmd_handle_cmds_duration_seconds_sum{type="wait"}[%ds])) / sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{type="wait"}[%ds])))`, seconds, seconds), endTime) tikv_coprocessor_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tikv_grpc_msg_duration_seconds_sum{type="coprocessor"}[%ds])) / sum(rate(tikv_grpc_msg_duration_seconds_count{type="coprocessor"}[%ds])))`, seconds, seconds), endTime) tikv_kv_get_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tikv_grpc_msg_duration_seconds_sum{type="kv_get"}[%ds])) / sum(rate(tikv_grpc_msg_duration_seconds_count{type="kv_get"}[%ds])))`, seconds, seconds), endTime) tikv_prewrite_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tikv_grpc_msg_duration_seconds_sum{type="kv_prewrite"}[%ds])) / sum(rate(tikv_grpc_msg_duration_seconds_count{type="kv_prewrite"}[%ds])))`, seconds, seconds), endTime) apply_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tikv_raftstore_apply_log_duration_seconds_sum[%ds])) / sum(rate(tikv_raftstore_apply_log_duration_seconds_count[%ds])))`, seconds, seconds), endTime) append_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tikv_raftstore_append_log_duration_seconds_sum[%ds])) / sum(rate(tikv_raftstore_append_log_duration_seconds_count[%ds])))`, seconds, seconds), endTime) commit_avg, _ := getMetric(promAPI, fmt.Sprintf(`1000*(sum(rate(tikv_raftstore_commit_log_duration_seconds_sum[%ds])) / sum(rate(tikv_raftstore_commit_log_duration_seconds_count[%ds])))`, seconds, seconds), endTime) idle_avg_not_in_txn_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tidb_server_conn_idle_duration_seconds_sum{in_txn='0'}[1m])) / sum(rate(tidb_server_conn_idle_duration_seconds_count{in_txn='0'}[1m])))[%ds:])`, seconds), endTime) idle_avg_in_txn_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tidb_server_conn_idle_duration_seconds_sum{in_txn='1'}[1m])) / sum(rate(tidb_server_conn_idle_duration_seconds_count{in_txn='1'}[1m])))[%ds:])`, seconds), endTime) parse_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tidb_session_parse_duration_seconds_sum{sql_type="general"}[1m])) / sum(rate(tidb_session_parse_duration_seconds_count{sql_type="general"}[1m])))[%ds:])`, seconds), endTime) compile_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tidb_session_compile_duration_seconds_sum{sql_type="general"}[1m])) / sum(rate(tidb_session_compile_duration_seconds_count{sql_type="general"}[1m])))[%ds:])`, seconds), endTime) execute_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tidb_session_execute_duration_seconds_sum{sql_type="general"}[1m])) / sum(rate(tidb_session_execute_duration_seconds_count{sql_type="general"}[1m])))[%ds:])`, seconds), endTime) tidb_prewrite_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tidb_tikvclient_request_seconds_sum{type="Prewrite"}[1m])) / sum(rate(tidb_tikvclient_request_seconds_count{type="Prewrite"}[1m])))[%ds:])`, seconds), endTime) tidb_kv_get_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tidb_tikvclient_request_seconds_sum{type="Get"}[1m])) / sum(rate(tidb_tikvclient_request_seconds_count{type="Get"}[1m])))[%ds:])`, seconds), endTime) tidb_coprocessor_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tidb_tikvclient_request_seconds_sum{type="Cop"}[1m])) / sum(rate(tidb_tikvclient_request_seconds_count{type="Cop"}[1m])))[%ds:])`, seconds), endTime) pd_tso_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(pd_client_cmd_handle_cmds_duration_seconds_sum{type="wait"}[1m])) / sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{type="wait"}[1m])))[%ds:])`, seconds), endTime) tikv_coprocessor_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tikv_grpc_msg_duration_seconds_sum{type="coprocessor"}[1m])) / sum(rate(tikv_grpc_msg_duration_seconds_count{type="coprocessor"}[1m])))[%ds:])`, seconds), endTime) tikv_kv_get_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tikv_grpc_msg_duration_seconds_sum{type="kv_get"}[1m])) / sum(rate(tikv_grpc_msg_duration_seconds_count{type="kv_get"}[1m])))[%ds:])`, seconds), endTime) tikv_prewrite_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tikv_grpc_msg_duration_seconds_sum{type="kv_prewrite"}[1m])) / sum(rate(tikv_grpc_msg_duration_seconds_count{type="kv_prewrite"}[1m])))[%ds:])`, seconds), endTime) apply_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tikv_raftstore_apply_log_duration_seconds_sum[1m])) / sum(rate(tikv_raftstore_apply_log_duration_seconds_count[1m])))[%ds:])`, seconds), endTime) append_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tikv_raftstore_append_log_duration_seconds_sum[1m])) / sum(rate(tikv_raftstore_append_log_duration_seconds_count[1m])))[%ds:])`, seconds), endTime) commit_avg_stddev, _ := getMetric(promAPI, fmt.Sprintf(`stddev_over_time((sum(rate(tikv_raftstore_commit_log_duration_seconds_sum[1m])) / sum(rate(tikv_raftstore_commit_log_duration_seconds_count[1m])))[%ds:])`, seconds), endTime) return avgQps, avgTps, avgP90, avgP99, avgP999, avgLatency, maxLatency, minQps, minTps, minP90, minP99, minP999, maxQps, maxTps, maxP90, maxP99, maxP999, idle_80_not_in_txn, idle_80_in_txn, parse_80, compile_80, execute_80, tidb_prewrite_80, tidb_kv_get_80, tidb_coprocessor_80, tidb_cpu_avg, tidb_cpu_delta_avg, plan_cache_ops, pd_tso_80, tikv_cpu_avg, tikv_cpu_delta_avg, tikv_coprocessor_80, tikv_kv_get_80, tikv_prewrite_80, apply_80, append_80, commit_80, idle_99_not_in_txn, idle_99_in_txn, parse_99, compile_99, execute_99, tidb_prewrite_99, tidb_kv_get_99, tidb_coprocessor_99, pd_tso_99, tikv_coprocessor_99, tikv_kv_get_99, tikv_prewrite_99, apply_99, append_99, commit_99, p80_latency, qps_stddev, p80_latency_stddev, p99_latency_stddev, p999_latency_stddev, idle_80_not_in_txn_stddev, idle_80_in_txn_stddev, parse_80_stddev, compile_80_stddev, execute_80_stddev, tidb_prewrite_80_stddev, tidb_kv_get_80_stddev, tidb_coprocessor_80_stddev, pd_tso_80_stddev, tikv_coprocessor_80_stddev, tikv_kv_get_80_stddev, tikv_prewrite_80_stddev, apply_80_stddev, append_80_stddev, commit_80_stddev, idle_99_not_in_txn_stddev, idle_99_in_txn_stddev, parse_99_stddev, compile_99_stddev, execute_99_stddev, tidb_prewrite_99_stddev, tidb_kv_get_99_stddev, tidb_coprocessor_99_stddev, pd_tso_99_stddev, tikv_coprocessor_99_stddev, tikv_kv_get_99_stddev, tikv_prewrite_99_stddev, apply_99_stddev, append_99_stddev, commit_99_stddev, idle_avg_not_in_txn, idle_avg_in_txn, parse_avg, compile_avg, execute_avg, tidb_prewrite_avg, tidb_kv_get_avg, tidb_coprocessor_avg, pd_tso_avg, tikv_coprocessor_avg, tikv_kv_get_avg, tikv_prewrite_avg, apply_avg, append_avg, commit_avg, idle_avg_not_in_txn_stddev, idle_avg_in_txn_stddev, parse_avg_stddev, compile_avg_stddev, execute_avg_stddev, tidb_prewrite_avg_stddev, tidb_kv_get_avg_stddev, tidb_coprocessor_avg_stddev, pd_tso_avg_stddev, tikv_coprocessor_avg_stddev, tikv_kv_get_avg_stddev, tikv_prewrite_avg_stddev, apply_avg_stddev, append_avg_stddev, commit_avg_stddev } func promReadyCheck(prometheusAddr string) { deployBackOff := backoff.NewExponentialBackOff() deployBackOff.InitialInterval = 5 * time.Second deployBackOff.MaxInterval = 5 * time.Minute deployBackOff.Multiplier = 3 deployBackOff.MaxElapsedTime = 10 * time.Minute retryable := func() error { promAPI, err := newPrometheusAPI(prometheusAddr) if err != nil { return err } _, err = getMetric(promAPI, "rate(tidb_server_query_total[60s])", time.Now()) if err != nil { log.Info("query error: ", zap.Error(err)) return err } return nil } backoff.Retry(retryable, deployBackOff) } func getMetric(promAPI v1.API, query string, ts time.Time) (float64, error) { // Record the current value of the query results. val, _, err := promAPI.Query(context.Background(), query, ts) if err != nil { return 0, err } //log.Info("Record the query and value", //ap.String("query", query), //ap.String("value", format.Object(val, 0)), //ap.Time("ts", ts)) valMatrix := val.(model.Vector) // fix the Error 1690: constant NaN overflows double, the value from prom exepression is not a number // http://172.16.4.180:31714/artifacts/testground/plan-exec-600556-st7xs/plan-exec-600556-st7xs-428117755/main-logs if valMatrix.Len() == 0 { return 0, nil } else { value := float64(valMatrix[0].Value) if math.IsNaN(value) { return 0, nil } else { return value, nil } } } func renderTopo(benchType string, tidbs int, tikvs int, tiflashs int) string { tidbPrefix := "tidb_servers:" tikvPrefix := "tikv_servers:" tiflashPrefix := "tiflash_servers:" topoList := []string{} if tidbs > 0 { tidbTpl := ` - host: tidb-%d-peer` tidbList := []string{tidbPrefix} for i := 1; i <= tidbs; i++ { tidbList = append(tidbList, fmt.Sprintf(tidbTpl, i)) } topoList = append(topoList, strings.Join(tidbList, "\n")) } if tikvs > 0 { tikvTpl := ` - host: tikv-%d-peer` tikvList := []string{tikvPrefix} for i := 1; i <= tikvs; i++ { tikvList = append(tikvList, fmt.Sprintf(tikvTpl, i)) } topoList = append(topoList, strings.Join(tikvList, "\n")) } if tiflashs > 0 { tiflashTpl := ` - host: tiflash-%d-peer` tiflashList := []string{tiflashPrefix} for i := 1; i <= tiflashs; i++ { tiflashList = append(tiflashList, fmt.Sprintf(tiflashTpl, i)) } topoList = append(topoList, strings.Join(tiflashList, "\n")) } return strings.Join(topoList, "\n") } // Convert json string to map func JsonToMap(rawJson []byte) (map[string]interface{}, error) { m := make(map[string]interface{}) err := json.Unmarshal(rawJson, &m) if err != nil { return nil, err } return m, nil } // Convert map json string func MapToJson(m map[string]interface{}) ([]byte, error) { jsonByte, err := json.Marshal(m) if err != nil { return []byte{}, nil } return jsonByte, nil } // Convert yaml to toml func YamlToToml(config string) (string, error) { m := make(map[string]interface{}) if err := yaml.Unmarshal([]byte(config), &m); err != nil { return "", err } buf := new(bytes.Buffer) e := toml.NewEncoder(buf) if err := e.Encode(m); err != nil { return "", err } return buf.String(), nil }