package recordingrule import ( "strings" "testing" "github.com/stretchr/testify/assert" ) func Test_ReplacePanelExprByRecordingRuleMetrics(t *testing.T) { replaceStrGraph := "{ \"annotations\": { \"list\": [ { \"builtIn\": 1, \"datasource\": \"-- Grafana --\", \"enable\": true, \"hide\": true, \"iconColor\": \"rgba(0, 211, 255, 1)\", \"name\": \"Annotations & Alerts\", \"type\": \"dashboard\" } ] }, \"editable\": true, \"gnetId\": null, \"graphTooltip\": 0, \"id\": 23, \"links\": [], \"panels\": [ { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"fill\": 1, \"fillGradient\": 0, \"gridPos\": { \"h\": 9, \"w\": 12, \"x\": 0, \"y\": 0 }, \"hiddenSeries\": false, \"id\": 2, \"legend\": { \"avg\": false, \"current\": false, \"max\": false, \"min\": false, \"show\": true, \"total\": false, \"values\": false }, \"lines\": true, \"linewidth\": 1, \"nullPointMode\": \"null\", \"options\": { \"alertThreshold\": true }, \"percentage\": false, \"pluginVersion\": \"7.5.15\", \"pointradius\": 2, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"exemplar\": true, \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{cluster_id=~\\\".*$tidb_cluster\\\", sql_type!=\\\"internal\\\"}[2m])) by (le))\", \"interval\": \"\", \"legendFormat\": \"\", \"refId\": \"A\" } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Panel Title\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"refresh\": \"30s\", \"schemaVersion\": 27, \"style\": \"dark\", \"tags\": [], \"templating\": { \"list\": [] }, \"time\": { \"from\": \"now-6h\", \"to\": \"now\" }, \"timepicker\": {}, \"timezone\": \"\", \"title\": \"test-dashboard\", \"uid\": \"PDtndvO4z\", \"version\": 2}" replaceStrRows := "{ \"annotations\": { \"list\": [ { \"builtIn\": 1, \"datasource\": \"tidbcloud-1372813089187041280\", \"enable\": true, \"hide\": true, \"iconColor\": \"rgba(0, 211, 255, 1)\", \"name\": \"Annotations & Alerts\", \"type\": \"dashboard\" } ] }, \"editable\": true, \"fiscalYearStartMonth\": 0, \"gnetId\": null, \"graphTooltip\": 0, \"id\": 121, \"iteration\": 1668649268040, \"links\": [], \"liveNow\": false, \"panels\": [ { \"collapsed\": true, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 0 }, \"id\": 33, \"panels\": [ { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 0.5 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"Duration alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"TiDB query durations by histogram buckets with different percents\", \"fill\": 1, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 1 }, \"id\": 3, \"legend\": { \"alignAsTable\": false, \"avg\": false, \"current\": false, \"hideEmpty\": false, \"max\": false, \"min\": false, \"show\": true, \"total\": false, \"values\": false }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"instant\": false, \"intervalFactor\": 2, \"legendFormat\": \"999\", \"refId\": \"A\", \"step\": 10 }, { \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 3, \"legendFormat\": \"99\", \"refId\": \"B\", \"step\": 15 }, { \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"95\", \"refId\": \"C\" }, { \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"80\", \"refId\": \"D\" } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 0.5 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Duration\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"s\", \"label\": null, \"logBase\": 2, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 2 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"Get Token Duration alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"Duration (us) for getting token, it should be small until concurrency limit is reached.\", \"fill\": 1, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 12, \"y\": 1 }, \"id\": 31, \"legend\": { \"avg\": false, \"current\": false, \"max\": false, \"min\": false, \"show\": true, \"total\": false, \"values\": false }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 1, \"legendFormat\": \"99\", \"refId\": \"A\" } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 2 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"99% Get Token Duration\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"µs\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 500 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"Connection Count alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"TiDB current connection counts\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 8 }, \"id\": 1, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"max\": false, \"min\": false, \"rightSide\": true, \"show\": true, \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [ { \"alias\": \"total\", \"fill\": 0, \"lines\": false } ], \"spaceLength\": 10, \"stack\": true, \"steppedLine\": true, \"targets\": [ { \"expr\": \"tidb_server_connections{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{instance}}\", \"refId\": \"A\", \"step\": 10 }, { \"expr\": \"sum(tidb_server_connections{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"})\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"total\", \"refId\": \"B\", \"step\": 10 } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 500 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Connection Count\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": \"0\", \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 1000000000 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" }, { \"evaluator\": { \"params\": [ 3000000000 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"Heap Memory Usage alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"TiDB heap memory size in use \", \"fill\": 0, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 12, \"y\": 8 }, \"id\": 2, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"hideEmpty\": true, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"go_memstats_heap_inuse_bytes{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", component=~\\\"tidb.*\\\"}\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{instance}}-{{component}}\", \"metric\": \"go_memstats_heap_inuse_bytes\", \"refId\": \"A\", \"step\": 10 } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 1000000000 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Heap Memory Usage\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"bytes\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": \"0\", \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": false } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"repeat\": null, \"title\": \"TiDB-Server\", \"type\": \"row\" }, { \"collapsed\": true, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 1 }, \"id\": 34, \"panels\": [ { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 0.01 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"Parse Duration alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": null, \"description\": \"The time cost of parsing SQL to AST\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 2 }, \"id\": 4, \"interval\": \"\", \"legend\": { \"alignAsTable\": true, \"avg\": true, \"current\": true, \"hideEmpty\": true, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sort\": null, \"sortDesc\": null, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le, sql_type))\", \"format\": \"time_series\", \"instant\": false, \"intervalFactor\": 2, \"legendFormat\": \"{{sql_type}}\", \"refId\": \"A\", \"step\": 30 } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 0.01 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"99% Parse Duration\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"cumulative\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"decimals\": null, \"format\": \"s\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": \"0\", \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": false } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"repeat\": null, \"title\": \"Parse\", \"type\": \"row\" }, { \"collapsed\": true, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 2 }, \"id\": 35, \"panels\": [ { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 0.03 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"Compile Duration alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": null, \"description\": \"The time cost of building the query plan\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 24, \"x\": 0, \"y\": 3 }, \"id\": 5, \"interval\": \"\", \"legend\": { \"alignAsTable\": true, \"avg\": true, \"current\": true, \"hideEmpty\": true, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sort\": null, \"sortDesc\": null, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le, sql_type))\", \"format\": \"time_series\", \"instant\": false, \"intervalFactor\": 2, \"legendFormat\": \"{{sql_type}}\", \"refId\": \"A\", \"step\": 30 } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 0.03 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"99% Compile Duration\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"cumulative\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"decimals\": null, \"format\": \"s\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": \"0\", \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": false } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"repeat\": null, \"title\": \"Compile\", \"type\": \"row\" }, { \"collapsed\": true, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 3 }, \"id\": 36, \"panels\": [ { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"Bucketed histogram of transaction execution durations, including retry\", \"fill\": 1, \"gridPos\": { \"h\": 7, \"w\": 8, \"x\": 0, \"y\": 4 }, \"id\": 30, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": false, \"max\": false, \"min\": false, \"rightSide\": true, \"show\": true, \"total\": false, \"values\": false }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le, sql_type))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"99-{{sql_type}}\", \"refId\": \"A\" }, { \"expr\": \"histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le, sql_type))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"95-{{sql_type}}\", \"refId\": \"B\" }, { \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le, sql_type))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"80-{{sql_type}}\", \"refId\": \"C\" } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Duration\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"s\", \"label\": null, \"logBase\": 2, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 500 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"Transaction Statement Num alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"TiDB statements numbers within one transaction. Internal means TiDB inner transaction\", \"fill\": 1, \"gridPos\": { \"h\": 7, \"w\": 8, \"x\": 8, \"y\": 4 }, \"id\": 6, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": false, \"max\": false, \"min\": false, \"rightSide\": true, \"show\": true, \"total\": false, \"values\": false }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_transaction_statement_num_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le, sql_type))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"99-{{sql_type}}\", \"refId\": \"A\" }, { \"expr\": \"histogram_quantile(0.80, sum(rate(tidb_session_transaction_statement_num_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le, sql_type))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"80-{{sql_type}}\", \"refId\": \"B\" } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 500 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Transaction Statement Num\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 3 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"Transaction Retry Num alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"TiDB transaction retry histogram bucket statistics\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 8, \"x\": 16, \"y\": 4 }, \"id\": 7, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": false, \"max\": false, \"min\": false, \"rightSide\": true, \"show\": true, \"total\": false, \"values\": false }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"100\", \"refId\": \"A\", \"step\": 10 }, { \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_session_retry_num_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"99\", \"refId\": \"B\" }, { \"expr\": \"histogram_quantile(0.90, sum(rate(tidb_session_retry_num_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"90\", \"refId\": \"C\" } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 3 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Transaction Retry Num\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": \"0\", \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"repeat\": null, \"title\": \"Transaction\", \"type\": \"row\" }, { \"collapsed\": true, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 4 }, \"id\": 37, \"panels\": [ { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"kv command durations statistics by command type\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 5 }, \"id\": 10, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 2, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.999, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", type=~\\\"get|batch_get|seek|seek_reverse\\\"}[2m])) by (le, type))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{type}}\", \"refId\": \"A\", \"step\": 40 } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"KV Cmd Duration 9999\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"cumulative\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"s\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"kv command durations statistics by command type\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 12, \"y\": 5 }, \"id\": 11, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"hideEmpty\": true, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", type=~\\\"get|batch_get|seek|seek_reverse\\\"}[2m])) by (le, type))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{type}}\", \"refId\": \"A\", \"step\": 40 } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"KV Cmd Duration 99\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"cumulative\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"s\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 500 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"Lock Resolve OPS alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"lock resolve times\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 12 }, \"id\": 8, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": false, \"hideEmpty\": true, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 2, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"sum(rate(tidb_tikvclient_lock_resolver_actions_total{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (type)\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{type}}\", \"metric\": \"tidb_tikvclient_lock_resolver_actions_total{}\", \"refId\": \"A\", \"step\": 40 } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 500 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Lock Resolve OPS\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"cumulative\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": \"0\", \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"kv command durations statistics by command type\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 12, \"y\": 12 }, \"id\": 79, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"hideEmpty\": true, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[5m])) by (le, type))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{type}}\", \"refId\": \"A\", \"step\": 40 } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"99% KV Backoff Duration\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"cumulative\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"s\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 500 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"KV Backoff OPS alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"kv storage backoff times\", \"fill\": 1, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 19 }, \"id\": 9, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": false, \"hideEmpty\": true, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"total\": true, \"values\": true }, \"lines\": true, \"linewidth\": 2, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"sum(rate(tidb_tikvclient_backoff_seconds_count{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (type)\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{type}}\", \"refId\": \"A\" } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 500 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"KV Backoff OPS\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"repeat\": null, \"title\": \"KV\", \"type\": \"row\" }, { \"collapsed\": true, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 5 }, \"id\": 38, \"panels\": [ { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"The duration of a client calling GetTSAsync until received the TS result.\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 6 }, \"id\": 12, \"legend\": { \"avg\": false, \"current\": false, \"max\": false, \"min\": false, \"show\": true, \"total\": false, \"values\": false }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", type=\\\"tso\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"999\", \"refId\": \"A\", \"step\": 10 }, { \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", type=\\\"tso\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"99\", \"refId\": \"B\" }, { \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", type=\\\"tso\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"90\", \"refId\": \"C\" } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"PD TSO Wait Duration\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"cumulative\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"s\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 0.03 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"PD TSO RPC Duration alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"description\": \"The duration of a client sending TSO request until received the response.\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 12, \"y\": 6 }, \"id\": 13, \"legend\": { \"avg\": false, \"current\": false, \"max\": false, \"min\": false, \"show\": true, \"total\": false, \"values\": false }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", type=\\\"tso\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"999\", \"refId\": \"A\", \"step\": 10 }, { \"expr\": \"histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", type=\\\"tso\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"99\", \"refId\": \"B\" }, { \"expr\": \"histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", type=\\\"tso\\\"}[2m])) by (le))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"90\", \"refId\": \"C\" } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 0.03 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"PD TSO RPC Duration\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"cumulative\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"s\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"repeat\": null, \"title\": \"PD Client\", \"type\": \"row\" }, { \"collapsed\": true, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 6 }, \"id\": 39, \"panels\": [ { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 1, \"description\": \"The execution time of gRPC message\", \"fill\": 1, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 7 }, \"id\": 29, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"hideEmpty\": false, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sideWidth\": null, \"sort\": \"current\", \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", type=~\\\"kv_get|kv_batch_get|coprocessor\\\"}[5m])) by (le, type))\", \"format\": \"time_series\", \"interval\": \"\", \"intervalFactor\": 2, \"legendFormat\": \"{{type}}\", \"refId\": \"A\", \"step\": 10 } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"99% gRPC message duration\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"s\", \"label\": null, \"logBase\": 2, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 3.6 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"frequency\": \"60s\", \"handler\": 1, \"name\": \"gRPC poll CPU alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 1, \"description\": \"The CPU utilization of gRPC\", \"fill\": 1, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 12, \"y\": 7 }, \"id\": 14, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sideWidth\": null, \"sort\": \"current\", \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", name=~\\\"grpc.*\\\"}[2m])) by (instance, name)\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{ instance }} - {{name}}\", \"refId\": \"A\", \"step\": 4 } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 3.6 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"gRPC poll CPU\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"percentunit\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"repeat\": null, \"title\": \"gRPC\", \"type\": \"row\" }, { \"collapsed\": true, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 7 }, \"id\": 47, \"panels\": [ { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 3.6 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"for\": \"5m\", \"frequency\": \"1m\", \"handler\": 1, \"name\": \"Storage ReadPool CPU alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 1, \"description\": \"The CPU utilization of readpool\", \"editable\": true, \"error\": false, \"fill\": 0, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 8 }, \"id\": 57, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"hideEmpty\": true, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sideWidth\": null, \"sort\": \"current\", \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", name=~\\\"store_read.*\\\"}[2m])) by (instance)\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{instance}}\", \"metric\": \"tikv_thread_cpu_seconds_total\", \"refId\": \"A\", \"step\": 4 } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 3.6 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Storage ReadPool CPU\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"percentunit\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"title\": \"Storage\", \"type\": \"row\" }, { \"collapsed\": true, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 8 }, \"id\": 49, \"panels\": [ { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 0.05 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"for\": \"5m\", \"frequency\": \"1m\", \"handler\": 1, \"name\": \"Wait duration alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 1, \"description\": \"The time consumed when coprocessor requests are wait for being handled\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 9 }, \"id\": 53, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sort\": \"current\", \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le,req))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{req}}-100%\", \"refId\": \"D\" }, { \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le,req))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{req}}-99%\", \"refId\": \"A\", \"step\": 4 } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 0.05 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Wait duration\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"cumulative\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"decimals\": 1, \"format\": \"s\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": \"0\", \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": false } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 1, \"description\": \"The time consumed when handling coprocessor requests\", \"editable\": true, \"error\": false, \"fill\": 1, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 12, \"y\": 9 }, \"id\": 51, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sort\": \"current\", \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le,req))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{req}}-100%\", \"refId\": \"E\" }, { \"expr\": \"histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[2m])) by (le,req))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"{{req}}-99%\", \"refId\": \"A\", \"step\": 4 } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Handle duration\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 1, \"value_type\": \"cumulative\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"decimals\": 1, \"format\": \"s\", \"label\": \"\", \"logBase\": 1, \"max\": null, \"min\": \"0\", \"show\": true }, { \"decimals\": 1, \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": false } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"alert\": { \"conditions\": [ { \"evaluator\": { \"params\": [ 7.2 ], \"type\": \"gt\" }, \"operator\": { \"type\": \"and\" }, \"query\": { \"params\": [ \"A\", \"5m\", \"now\" ] }, \"reducer\": { \"params\": [], \"type\": \"avg\" }, \"type\": \"query\" } ], \"executionErrorState\": \"alerting\", \"for\": \"5m\", \"frequency\": \"1m\", \"handler\": 1, \"name\": \"Coprocessor CPU alert\", \"noDataState\": \"no_data\", \"notifications\": [] }, \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 1, \"description\": \"The CPU utilization of coprocessor\", \"editable\": true, \"error\": false, \"fill\": 0, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 12, \"x\": 0, \"y\": 16 }, \"id\": 55, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"hideEmpty\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sideWidth\": null, \"sort\": \"current\", \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", name=~\\\"cop_.*\\\"}[2m])) by (instance)\", \"format\": \"time_series\", \"interval\": \"\", \"intervalFactor\": 2, \"legendFormat\": \"{{instance}}\", \"refId\": \"A\", \"step\": 4 } ], \"thresholds\": [ { \"colorMode\": \"critical\", \"fill\": true, \"line\": true, \"op\": \"gt\", \"value\": 7.2 } ], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Coprocessor CPU\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"cumulative\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"percentunit\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"title\": \"Coprocessor\", \"type\": \"row\" }, { \"collapsed\": true, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 9 }, \"id\": 45, \"panels\": [ { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 1, \"description\": \"The time consumed when executing get operations\", \"fill\": 0, \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 0, \"y\": 10 }, \"id\": 77, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sideWidth\": null, \"sort\": \"current\", \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"max(tikv_engine_get_micro_seconds{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\",type=\\\"get_max\\\"})\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"max\", \"refId\": \"A\", \"step\": 10 }, { \"expr\": \"max(tikv_engine_get_micro_seconds{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\",type=\\\"get_percentile99\\\"})\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"99%\", \"refId\": \"B\", \"step\": 10 }, { \"expr\": \"max(tikv_engine_get_micro_seconds{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\",type=\\\"get_percentile95\\\"})\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"95%\", \"refId\": \"C\", \"step\": 10 }, { \"expr\": \"max(tikv_engine_get_micro_seconds{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\",type=\\\"get_average\\\"})\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"avg\", \"refId\": \"D\", \"step\": 10 } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Get duration\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"µs\", \"label\": null, \"logBase\": 2, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 1, \"description\": \"The count of get operations\", \"fill\": 1, \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 12, \"y\": 10 }, \"id\": 75, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sideWidth\": null, \"sort\": \"current\", \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"sum(rate(tikv_engine_memtable_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"memtable_hit\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"memtable\", \"metric\": \"\", \"refId\": \"B\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=~\\\"block_cache_data_hit|block_cache_filter_hit\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"block_cache\", \"metric\": \"\", \"refId\": \"E\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_get_served{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"get_hit_l0\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"l0\", \"refId\": \"A\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_get_served{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"get_hit_l1\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"l1\", \"refId\": \"C\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_get_served{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"get_hit_l2_and_up\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"l2_and_up\", \"refId\": \"F\", \"step\": 10 } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Get operations\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"ops\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 1, \"description\": \"The time consumed when executing seek operation\", \"fill\": 0, \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 0, \"y\": 18 }, \"id\": 73, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sideWidth\": null, \"sort\": \"current\", \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"max(tikv_engine_seek_micro_seconds{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\",type=\\\"seek_max\\\"})\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"max\", \"refId\": \"A\", \"step\": 10 }, { \"expr\": \"max(tikv_engine_seek_micro_seconds{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\",type=\\\"seek_percentile99\\\"})\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"99%\", \"refId\": \"B\", \"step\": 10 }, { \"expr\": \"max(tikv_engine_seek_micro_seconds{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\",type=\\\"seek_percentile95\\\"})\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"95%\", \"refId\": \"C\", \"step\": 10 }, { \"expr\": \"max(tikv_engine_seek_micro_seconds{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\",type=\\\"seek_average\\\"})\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"avg\", \"refId\": \"D\", \"step\": 10 } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Seek duration\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"µs\", \"label\": null, \"logBase\": 2, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 1, \"description\": \"The count of seek operations\", \"fill\": 1, \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 12, \"y\": 18 }, \"id\": 71, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sideWidth\": null, \"sort\": \"current\", \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"nullPointMode\": \"null\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"sum(rate(tikv_engine_locate{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"number_db_seek\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"seek\", \"metric\": \"\", \"refId\": \"A\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_locate{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"number_db_seek_found\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"seek_found\", \"metric\": \"\", \"refId\": \"B\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_locate{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"number_db_next\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"next\", \"metric\": \"\", \"refId\": \"C\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_locate{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"number_db_next_found\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"next_found\", \"metric\": \"\", \"refId\": \"D\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_locate{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"number_db_prev\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"prev\", \"metric\": \"\", \"refId\": \"E\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_locate{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"number_db_prev_found\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"prev_found\", \"metric\": \"\", \"refId\": \"F\", \"step\": 10 } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Seek operations\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"ops\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"short\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 1, \"description\": \"The hit rate of block cache\", \"fill\": 0, \"gridPos\": { \"h\": 8, \"w\": 12, \"x\": 0, \"y\": 26 }, \"id\": 69, \"legend\": { \"alignAsTable\": true, \"avg\": false, \"current\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sideWidth\": null, \"sort\": \"current\", \"sortDesc\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 1, \"links\": [], \"maxPerRow\": 2, \"nullPointMode\": \"connected\", \"percentage\": false, \"pointradius\": 5, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"expr\": \"sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", instance=~\\\"$instance\\\", db=\\\"$db\\\", type=\\\"block_cache_hit\\\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"$db\\\", type=\\\"block_cache_hit\\\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"$db\\\", type=\\\"block_cache_miss\\\"}[2m])))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"all\", \"metric\": \"\", \"refId\": \"A\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"block_cache_data_hit\\\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"block_cache_data_hit\\\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"block_cache_data_miss\\\"}[2m])))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"data\", \"metric\": \"\", \"refId\": \"D\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"block_cache_filter_hit\\\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"block_cache_filter_hit\\\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"block_cache_filter_miss\\\"}[2m])))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"filter\", \"metric\": \"\", \"refId\": \"B\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"block_cache_index_hit\\\"}[2m])) / (sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"block_cache_index_hit\\\"}[2m])) + sum(rate(tikv_engine_cache_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"block_cache_index_miss\\\"}[2m])))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"index\", \"metric\": \"\", \"refId\": \"C\", \"step\": 10 }, { \"expr\": \"sum(rate(tikv_engine_bloom_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"bloom_prefix_useful\\\"}[2m])) / sum(rate(tikv_engine_bloom_efficiency{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\", db=\\\"kv\\\", type=\\\"bloom_prefix_checked\\\"}[2m]))\", \"format\": \"time_series\", \"intervalFactor\": 2, \"legendFormat\": \"bloom prefix\", \"metric\": \"\", \"refId\": \"E\", \"step\": 10 } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Block cache hit\", \"tooltip\": { \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"percentunit\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": \"0\", \"show\": true }, { \"format\": \"ops\", \"label\": null, \"logBase\": 1, \"max\": null, \"min\": null, \"show\": false } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"repeat\": null, \"title\": \"RocksDB-KV\", \"type\": \"row\" }, { \"collapsed\": true, \"datasource\": null, \"fieldConfig\": { \"defaults\": {}, \"overrides\": [] }, \"gridPos\": { \"h\": 1, \"w\": 24, \"x\": 0, \"y\": 10 }, \"id\": 61, \"panels\": [ { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 2, \"description\": \"Shows average latency for Reads and Writes IO Devices. Higher than typical latency for highly loaded storage indicates saturation (overload) and is frequent cause of performance problems. Higher than normal latency also can indicate internal storage problems.\", \"editable\": true, \"error\": false, \"fill\": 2, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 24, \"x\": 0, \"y\": 11 }, \"id\": 59, \"legend\": { \"alignAsTable\": true, \"avg\": true, \"current\": false, \"hideEmpty\": true, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sort\": null, \"sortDesc\": null, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 2, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 1, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"calculatedInterval\": \"2m\", \"datasourceErrors\": {}, \"errors\": {}, \"expr\": \"irate(node_disk_read_time_seconds_total{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[5m]) / irate(node_disk_reads_completed_total{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[5m])\", \"format\": \"time_series\", \"interval\": \"\", \"intervalFactor\": 1, \"legendFormat\": \"Read: {{ instance }} - {{ device }}\", \"metric\": \"\", \"refId\": \"A\", \"step\": 300, \"target\": \"\" } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Disk Latency\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"s\", \"label\": \"\", \"logBase\": 2, \"max\": null, \"min\": null, \"show\": true }, { \"format\": \"s\", \"label\": \"\", \"logBase\": 1, \"max\": null, \"min\": null, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 2, \"description\": \"Shows amount of physical IOs (reads and writes) different devices are serving. Spikes in number of IOs served often corresponds to performance problems due to IO subsystem overload.\", \"editable\": true, \"error\": false, \"fill\": 2, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 24, \"x\": 0, \"y\": 18 }, \"id\": 63, \"legend\": { \"alignAsTable\": true, \"avg\": true, \"current\": false, \"hideEmpty\": false, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sort\": null, \"sortDesc\": null, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 2, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 1, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"calculatedInterval\": \"2m\", \"datasourceErrors\": {}, \"errors\": {}, \"expr\": \"irate(node_disk_reads_completed_total{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[5m])\", \"format\": \"time_series\", \"interval\": \"\", \"intervalFactor\": 1, \"legendFormat\": \"Read: {{ instance }} - {{ device }}\", \"metric\": \"\", \"refId\": \"A\", \"step\": 300, \"target\": \"\" } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Disk Operations\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"iops\", \"label\": \"\", \"logBase\": 1, \"max\": null, \"min\": 0, \"show\": true }, { \"format\": \"short\", \"label\": \"\", \"logBase\": 1, \"max\": null, \"min\": 0, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 2, \"description\": \"Shows volume of reads and writes the storage is handling. This can be better measure of IO capacity usage for network attached and SSD storage as it is often bandwidth limited. Amount of data being written to the disk can be used to estimate Flash storage life time.\", \"editable\": true, \"error\": false, \"fill\": 2, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 24, \"x\": 0, \"y\": 25 }, \"id\": 65, \"legend\": { \"alignAsTable\": true, \"avg\": true, \"current\": false, \"hideEmpty\": false, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"sort\": null, \"sortDesc\": null, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 2, \"links\": [], \"nullPointMode\": \"null as zero\", \"percentage\": false, \"pointradius\": 1, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"calculatedInterval\": \"2m\", \"datasourceErrors\": {}, \"errors\": {}, \"expr\": \"irate(node_disk_read_bytes_total{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[5m])\", \"format\": \"time_series\", \"interval\": \"\", \"intervalFactor\": 1, \"legendFormat\": \"Read: {{ instance }} - {{ device }}\", \"metric\": \"\", \"refId\": \"A\", \"step\": 300, \"target\": \"\" } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Disk Bandwidth\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"Bps\", \"label\": \"\", \"logBase\": 1, \"max\": null, \"min\": 0, \"show\": true }, { \"format\": \"short\", \"label\": \"\", \"logBase\": 1, \"max\": null, \"min\": 0, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } }, { \"aliasColors\": {}, \"bars\": false, \"dashLength\": 10, \"dashes\": false, \"datasource\": \"tidbcloud-1372813089187041280\", \"decimals\": 2, \"description\": \"Shows how much disk was loaded for reads or writes as average number of outstanding requests at different period of time. High disk load is a good measure of actual storage utilization. Different storage types handle load differently - some will show latency increases on low loads others can handle higher load with no problems.\", \"editable\": true, \"error\": false, \"fill\": 2, \"grid\": {}, \"gridPos\": { \"h\": 7, \"w\": 24, \"x\": 0, \"y\": 32 }, \"id\": 67, \"legend\": { \"alignAsTable\": true, \"avg\": true, \"current\": false, \"hideEmpty\": false, \"hideZero\": true, \"max\": true, \"min\": false, \"rightSide\": true, \"show\": true, \"total\": false, \"values\": true }, \"lines\": true, \"linewidth\": 2, \"links\": [], \"nullPointMode\": \"null\", \"percentage\": false, \"pointradius\": 1, \"points\": false, \"renderer\": \"flot\", \"seriesOverrides\": [], \"spaceLength\": 10, \"stack\": false, \"steppedLine\": false, \"targets\": [ { \"calculatedInterval\": \"2m\", \"datasourceErrors\": {}, \"errors\": {}, \"expr\": \"irate(node_disk_read_time_seconds_total{k8s_cluster=\\\"$k8s_cluster\\\", cluster_id=~\\\".*$tidb_cluster\\\"}[5m])\", \"format\": \"time_series\", \"interval\": \"\", \"intervalFactor\": 1, \"legendFormat\": \"Read: {{instance}} - {{ device }}\", \"metric\": \"\", \"refId\": \"A\", \"step\": 300, \"target\": \"\" } ], \"thresholds\": [], \"timeFrom\": null, \"timeRegions\": [], \"timeShift\": null, \"title\": \"Disk Load\", \"tooltip\": { \"msResolution\": false, \"shared\": true, \"sort\": 0, \"value_type\": \"individual\" }, \"type\": \"graph\", \"xaxis\": { \"buckets\": null, \"mode\": \"time\", \"name\": null, \"show\": true, \"values\": [] }, \"yaxes\": [ { \"format\": \"short\", \"label\": \"\", \"logBase\": 1, \"max\": null, \"min\": 0, \"show\": true }, { \"format\": \"short\", \"label\": \"\", \"logBase\": 1, \"max\": null, \"min\": 0, \"show\": true } ], \"yaxis\": { \"align\": false, \"alignLevel\": null } } ], \"title\": \"Disk\", \"type\": \"row\" } ], \"refresh\": \"1m\", \"schemaVersion\": 32, \"style\": \"dark\", \"tags\": [], \"templating\": { \"list\": [ { \"allValue\": null, \"current\": { \"isNone\": true, \"selected\": false, \"text\": \"None\", \"value\": \"\" }, \"datasource\": \"tidbcloud-1372813089187041280\", \"definition\": \"\", \"description\": null, \"error\": null, \"hide\": 2, \"includeAll\": false, \"label\": \"K8s-cluster\", \"multi\": false, \"name\": \"k8s_cluster\", \"options\": [], \"query\": { \"query\": \"label_values(tikv_engine_block_cache_size_bytes, k8s_cluster)\", \"refId\": \"Prometheus-k8s_cluster-Variable-Query\" }, \"refresh\": 2, \"regex\": \"\", \"skipUrlSync\": false, \"sort\": 1, \"tagValuesQuery\": \"\", \"tagsQuery\": \"\", \"type\": \"query\", \"useTags\": false }, { \"allValue\": null, \"current\": { \"selected\": false, \"text\": \"1379661944579024068\", \"value\": \"1379661944579024068\" }, \"datasource\": \"Prometheus-fake\", \"definition\": \"label_values({component=\\\"pd\\\"}, tidb_cluster)\", \"description\": null, \"error\": null, \"hide\": 0, \"includeAll\": false, \"label\": \"tidb_cluster\", \"multi\": false, \"name\": \"tidb_cluster\", \"options\": [], \"query\": { \"query\": \"label_values({component=\\\"pd\\\"}, tidb_cluster)\", \"refId\": \"StandardVariableQuery\" }, \"refresh\": 2, \"regex\": \"\", \"skipUrlSync\": false, \"sort\": 0, \"type\": \"query\" }, { \"description\": null, \"error\": null, \"hide\": 2, \"label\": null, \"name\": \"org\", \"query\": \"\", \"skipUrlSync\": false, \"type\": \"constant\" }, { \"description\": null, \"error\": null, \"hide\": 2, \"label\": null, \"name\": \"tidb_version\", \"query\": \"\", \"skipUrlSync\": false, \"type\": \"constant\" }, { \"description\": null, \"error\": null, \"hide\": 2, \"label\": null, \"name\": \"org_type\", \"query\": \"\", \"skipUrlSync\": false, \"type\": \"constant\" } ] }, \"time\": { \"from\": \"now-2h\", \"to\": \"now\" }, \"timepicker\": { \"refresh_intervals\": [ \"5s\", \"10s\", \"30s\", \"1m\", \"5m\", \"15m\", \"30m\", \"1h\", \"2h\", \"1d\" ], \"time_options\": [ \"5m\", \"15m\", \"1h\", \"6h\", \"12h\", \"24h\", \"2d\", \"7d\", \"30d\" ] }, \"timezone\": \"\", \"title\": \"Performance-Read-Cluster-Clinic\", \"uid\": \"4aVOvxcWktidbcloud\", \"version\": 2}" t.Run("replace graph panel", func(t *testing.T) { replaceStr, _ := ReplacePanelExprByRecordingRuleMetrics(replaceStrGraph) checkStatus := strings.Contains(replaceStr, "tidb_server_handle_query_duration_seconds_bucket:le_sqltype_not:sum:rate") assert.True(t, checkStatus) }) t.Run("replace rows panel", func(t *testing.T) { replaceStr, _ := ReplacePanelExprByRecordingRuleMetrics(replaceStrRows) checkStatus := strings.Contains(replaceStr, "tidb_server_handle_query_duration_seconds_bucket:le:sum:rate") assert.True(t, checkStatus) checkStatusTiKV := strings.Contains(replaceStr, "tidb_tikvclient_txn_cmd_duration_seconds_bucket:sum:rate") assert.True(t, checkStatusTiKV) checkStatusSession := strings.Contains(replaceStr, "tidb_session_transaction_duration_seconds_bucket:le_sql_type:sum:rate") assert.True(t, checkStatusSession) }) }