{ "__inputs": [ { "name": "DS_TEST-CLUSTER", "label": "${DS_TEST-CLUSTER}", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "6.1.6" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "singlestat", "name": "Singlestat", "version": "" }, { "type": "panel", "id": "table", "name": "Table", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "${DS_TEST-CLUSTER}", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 1, "id": 31, "iteration": 1669018858346, "links": [], "panels": [ { "collapsed": false, "datasource": "${DS_TEST-CLUSTER}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 138, "panels": [], "title": "Performance Overview", "type": "row" }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Service Time Per Second, show service time distribution among different SQL types:\n1. Database time, the total time that the TiDB cluster is processing application requests.\n2. The service time of different SQL types.\n", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 1 }, "hiddenSeries": false, "id": 189, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:601", "alias": "database time", "bars": false, "color": "#FADE2A", "lines": true, "linewidth": 2, "stack": false }, { "$$hashKey": "object:637", "alias": "Select", "color": "#5794F2" }, { "$$hashKey": "object:645", "alias": "Commit", "color": "#96D98D" }, { "$$hashKey": "object:655", "alias": "Insert", "color": "#73BF69" }, { "$$hashKey": "object:700", "alias": "general", "color": "#F2495C" }, { "$$hashKey": "object:2605", "alias": "Update", "color": "#37872D" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(tidb_server_tokens{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"})", "hide": false, "interval": "", "legendFormat": "database time", "refId": "A" }, { "exemplar": true, "expr": "sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[1m])) by (sql_type)", "hide": false, "interval": "", "legendFormat": "{{sql_type}}", "refId": "G" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Database Time by SQL Type", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3528", "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:3529", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Service Time Per Second, show service time distribution among different SQL phases.\n1. Database time, the total time that the TiDB cluster is processing application requests.\n2. The service time of different SQL phases.\n", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 1 }, "hiddenSeries": false, "id": 187, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:205", "alias": "parse", "color": "#FF7383" }, { "$$hashKey": "object:216", "alias": "compile", "color": "#FF9830" }, { "$$hashKey": "object:227", "alias": "execute", "color": "#73BF69" }, { "$$hashKey": "object:364", "alias": "database time", "bars": false, "color": "#FADE2A", "lines": true, "linewidth": 2, "stack": false }, { "$$hashKey": "object:2194", "alias": "get token", "color": "#C4162A" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(tidb_server_tokens{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"})", "hide": false, "interval": "", "legendFormat": "database time", "refId": "A" }, { "exemplar": true, "expr": "sum(rate(tidb_session_parse_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m]))", "hide": false, "interval": "", "legendFormat": "parse", "refId": "D" }, { "exemplar": true, "expr": "sum(rate(tidb_session_compile_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m]))", "hide": false, "interval": "", "legendFormat": "compile", "refId": "E" }, { "exemplar": true, "expr": "sum(rate(tidb_session_execute_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m]))", "hide": false, "interval": "", "legendFormat": "execute", "refId": "F" }, { "exemplar": true, "expr": "sum(rate(tidb_server_get_token_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m]))/1000000", "hide": false, "instant": false, "interval": "", "legendFormat": "get token", "refId": "G" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Database Time by SQL Phase", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3528", "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:3529", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Service Time Per Second, show service time distribution among different KV/PD request:\n1. Execute time, the execute time in SQL Phase\n2. Service time of different KV/PD request ", "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 1 }, "hiddenSeries": false, "id": 190, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:1113", "alias": "tso_wait", "color": "#701313" }, { "$$hashKey": "object:1127", "alias": "Commit", "color": "#37872D" }, { "$$hashKey": "object:1139", "alias": "Prewrite", "color": "#73BF69" }, { "$$hashKey": "object:1144", "alias": "PessimisticLock", "color": "#b40606" }, { "$$hashKey": "object:2301", "alias": "Get", "color": "#5794F2" }, { "$$hashKey": "object:2313", "alias": "BatchGet", "color": "#1F60C4" }, { "$$hashKey": "object:2321", "alias": "Cop", "color": "#C0D8FF" }, { "$$hashKey": "object:2673", "alias": "Scan", "color": "#8778ee" }, { "$$hashKey": "object:2828", "alias": "execute time", "bars": false, "color": "#FADE2A", "lines": true, "linewidth": 2, "stack": false }, { "$$hashKey": "object:321", "alias": "tiflash_mpp", "color": "#8F3BB8" } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tidb_tikvclient_request_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", store!=\"0\"}[1m])) by (type)", "hide": false, "interval": "", "legendFormat": "{{type}}", "refId": "B" }, { "exemplar": true, "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m]))", "hide": false, "interval": "", "legendFormat": "tso_wait", "refId": "C" }, { "exemplar": true, "expr": "sum(rate(tidb_session_execute_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m]))", "hide": false, "interval": "", "legendFormat": "execute time", "refId": "F" }, { "exemplar": true, "expr": "sum(rate(tiflash_coprocessor_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) ", "hide": false, "interval": "", "legendFormat": "tiflash_mpp", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "SQL Execute Time Overview", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3528", "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:3529", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "TiDB statement statistics.\nBold red line on right Y axis for Failed Queries per second", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 8 }, "hiddenSeries": false, "id": 179, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:1069", "alias": "Failed", "color": "#C4162A", "linewidth": 2, "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 30 }, { "exemplar": true, "expr": "sum(rate(tidb_executor_statement_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m]))", "hide": false, "interval": "", "legendFormat": "Total", "refId": "B" }, { "exemplar": true, "expr": "sum(rate(tidb_server_execute_error_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) ", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Failed", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "QPS", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:822", "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:823", "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "TiDB command total statistics including both successful and failed ones", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 8 }, "hiddenSeries": false, "id": 178, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tidb_server_query_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPS By Type", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "TiDB plan cache hit total.", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 8 }, "hiddenSeries": false, "id": 91, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tidb_server_plan_cache_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "avg-hit", "refId": "A", "step": 30 }, { "exemplar": true, "expr": "sum(rate(tidb_server_plan_cache_miss_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-miss", "refId": "B", "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Queries Using Plan Cache OPS", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3416", "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:3417", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "kv/tso request total by command type", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 15 }, "hiddenSeries": false, "id": 180, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tidb_tikvclient_request_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 30 }, { "exemplar": true, "expr": "sum(rate(tidb_tikvclient_request_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m]))", "hide": false, "interval": "", "legendFormat": "kv request total", "refId": "B" }, { "exemplar": true, "expr": "sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m]))", "hide": false, "interval": "", "legendFormat": "tso - cmd", "refId": "C" }, { "exemplar": true, "expr": "sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m]))", "hide": false, "interval": "", "legendFormat": "tso - request", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KV/TSO Request OPS", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "kv request time by command source", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 15 }, "hiddenSeries": false, "id": 23763571995, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:243", "alias": "kv request total time", "bars": false, "color": "#FADE2A", "lines": true, "stack": false } ], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tidb_tikvclient_request_time_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type, source)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-{{source}}", "refId": "A", "step": 30 }, { "exemplar": true, "expr": "sum(rate(tidb_tikvclient_request_time_counter{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "hide": false, "interval": "", "legendFormat": "kv request total time", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KV Request Time By Source", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:178", "decimals": 0, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:179", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "1. CPU: CPU utilization for all TiDB instances.\n- CPU-Avg: avg(cpu utilization)\n- CPU-Delta: max(cpu utilization) - min(cpu utilization)\n- CPU-Max: max(cpu utilization)\n2. Max memory usage", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 22 }, "hiddenSeries": false, "id": 23763572004, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:371", "alias": "Mem-Max", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "avg(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "CPU-Avg", "refId": "A", "step": 30 }, { "exemplar": true, "expr": "max(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "CPU-Max", "refId": "C" }, { "exemplar": true, "expr": "(max(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m])) - min(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"}[1m])))", "hide": false, "interval": "", "legendFormat": "CPU-Delta", "refId": "B" }, { "exemplar": true, "expr": "max(tidb_server_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tidb\"})", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "CPU-Quota", "refId": "D", "step": 30 }, { "exemplar": true, "expr": "max(sum((process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=~\"tidb\"})) by (instance))", "hide": false, "interval": "", "legendFormat": "Mem-Max", "refId": "E" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "TiDB CPU/Memory", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:208", "decimals": null, "format": "percentunit", "label": "CPU (%)", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:209", "decimals": null, "format": "bytes", "label": "Memory", "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "1. CPU: CPU utilization for all TiKV instances.\n- CPU-Avg: avg(cpu utilization)\n- CPU-Delta: max(cpu utilization) - min(cpu utilization)\n- CPU-Max: max(cpu utilization)\n2. Max memory usage", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 22 }, "hiddenSeries": false, "id": 23763572003, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:371", "alias": "Mem-Max", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "avg(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=~\".*tikv\"}[1m]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "CPU-Avg", "refId": "A", "step": 30 }, { "exemplar": true, "expr": "max(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=~\".*tikv\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "CPU-Max", "refId": "C" }, { "exemplar": true, "expr": "(max(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=~\".*tikv\"}[1m])) - min(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=~\".*tikv\"}[1m])))", "hide": false, "interval": "", "legendFormat": "CPU-Delta", "refId": "B" }, { "exemplar": true, "expr": "max(tikv_server_cpu_cores_quota{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=~\".*tikv\"})", "hide": false, "interval": "", "legendFormat": "CPU-Quota", "refId": "D" }, { "exemplar": true, "expr": "max(sum((process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", job=~\".*tikv\"})) by (instance))", "hide": false, "interval": "", "legendFormat": "Mem-Max", "refId": "E" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "TiKV CPU/Memory", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:208", "decimals": null, "format": "percentunit", "label": "CPU (%)", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:209", "decimals": null, "format": "bytes", "label": "Memory", "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "1. CPU: CPU utilization for pd instances.\n- CPU-Max: max(cpu utilization)\n2. Max memory usage", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 22 }, "hiddenSeries": false, "id": 182, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:371", "alias": "Mem-Max", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "max(rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}[1m]))", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "CPU-Max", "refId": "C" }, { "exemplar": true, "expr": "max(pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"})", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "CPU-Quota", "refId": "A" }, { "exemplar": true, "expr": "max(process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"})", "hide": false, "interval": "", "legendFormat": "Mem-Max", "refId": "E" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "PD CPU/Memory", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:208", "decimals": null, "format": "percentunit", "label": "CPU (%)", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:209", "decimals": null, "format": "bytes", "label": "Memory", "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "Out traffic statistics From TiDB to the client.\nRead Flow between TiKV and Rocksdb.", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 30 }, "hiddenSeries": false, "id": 23763572000, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"Out|write\"}[1m])) by (type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "TiDB -> Client", "refId": "A", "step": 30 }, { "exemplar": true, "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=~\"bytes_read|iter_bytes_read\"}[1m]))", "hide": false, "interval": "", "legendFormat": "Rocksdb -> TiKV", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read Traffic", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:188", "decimals": null, "format": "Bps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:189", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "Write traffic statistics between from Client to TiDB , the txn write rate from TiDB to TiKV, and the write flow from TiKV to RocksDB.", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 30 }, "hiddenSeries": false, "id": 23763572002, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tidb_server_packet_io_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"In|read\"}[1m])) by (type)", "hide": false, "interval": "", "legendFormat": "Client -> TiDB", "refId": "C" }, { "exemplar": true, "expr": "sum(rate(tidb_tikvclient_txn_write_size_bytes_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (scope)", "hide": false, "interval": "", "legendFormat": "TiDB -> TiKV: {{scope}}", "refId": "D" }, { "exemplar": true, "expr": "sum(rate(tikv_engine_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=\"wal_file_bytes\"}[1m]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "TiKV -> Rocksdb", "refId": "A", "step": 30 }, { "exemplar": true, "expr": "sum(rate(tikv_engine_compaction_flow_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"bytes_read|bytes_written\"}[1m]))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "RocksDB Compaction", "refId": "B", "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Write Traffic", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:188", "decimals": null, "format": "Bps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:189", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "TiDB query durations", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 37 }, "hiddenSeries": false, "id": 80, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[1m])) / sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[1m]))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "avg", "refId": "D" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[1m])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { "exemplar": true, "expr": "sum(rate(tidb_server_handle_query_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[1m])) by (sql_type) / sum(rate(tidb_server_handle_query_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type!=\"internal\"}[1m])) by (sql_type)", "hide": false, "interval": "", "legendFormat": "avg-{{sql_type}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3308", "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:3309", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "TiDB connection idle durations", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 37 }, "hiddenSeries": false, "id": 171, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "(sum(rate(tidb_server_conn_idle_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", in_txn='1'}[1m])) / sum(rate(tidb_server_conn_idle_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", in_txn='1'}[1m])))", "hide": false, "interval": "", "legendFormat": "avg-in-txn", "refId": "C" }, { "exemplar": true, "expr": "(sum(rate(tidb_server_conn_idle_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", in_txn='0'}[1m])) / sum(rate(tidb_server_conn_idle_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", in_txn='0'}[1m])))", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-not-in-txn", "refId": "A", "step": 30 }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", in_txn='1'}[1m])) by (le,in_txn))", "hide": false, "interval": "", "legendFormat": "99-in-txn", "refId": "B" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tidb_server_conn_idle_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", in_txn='0'}[1m])) by (le,in_txn))", "hide": false, "interval": "", "legendFormat": "99-not-in-txn", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Connection Idle Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3364", "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:3365", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "TiDB current connection counts", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 37 }, "hiddenSeries": false, "id": 188, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:113", "alias": "disconnection/s", "color": "#C4162A", "linewidth": 2, "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "tidb_server_connections{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A", "step": 30 }, { "exemplar": true, "expr": "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"})", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "total", "refId": "B" }, { "exemplar": true, "expr": "sum(tidb_server_tokens{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"})", "hide": false, "interval": "", "legendFormat": "active connections", "refId": "D" }, { "exemplar": true, "expr": "sum(rate(tidb_server_disconnection_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", result=\"ok\"}[1m]))", "hide": false, "interval": "", "legendFormat": "disconnection/s", "refId": "E" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Connection Count", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3472", "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:3473", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "The time cost of parsing SQL to AST", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 43 }, "hiddenSeries": false, "id": 156, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "95", "refId": "B" }, { "exemplar": true, "expr": "(sum(rate(tidb_session_parse_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) / sum(rate(tidb_session_parse_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])))", "hide": false, "interval": "", "legendFormat": "avg", "refId": "C" }, { "expr": "histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "99", "refId": "D", "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Parse Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1955", "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:1956", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The time to build plan", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 43 }, "hiddenSeries": false, "id": 170, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "(sum(rate(tidb_session_compile_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) / sum(rate(tidb_session_compile_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])))", "hide": false, "interval": "", "legendFormat": "avg", "refId": "D" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "99", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Compile Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:283", "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:284", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "The time cost of executing the SQL which does not include the time to get the results of the query .", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 43 }, "hiddenSeries": false, "id": 169, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "95", "refId": "B" }, { "exemplar": true, "expr": "(sum(rate(tidb_session_execute_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) / sum(rate(tidb_session_execute_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])))", "hide": false, "interval": "", "legendFormat": "avg", "refId": "C" }, { "expr": "histogram_quantile(0.99, sum(rate(tidb_session_execute_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", sql_type=\"general\"}[1m])) by (le))", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "99", "refId": "A", "step": 30 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Execute Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2109", "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:2110", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "tidb avg kv request duration", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 43 }, "hiddenSeries": false, "id": 172, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tidb_tikvclient_request_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", store!=\"0\"}[1m])) by (type)/ sum(rate(tidb_tikvclient_request_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", store!=\"0\"}[1m])) by (type)", "hide": false, "interval": "", "legendFormat": "{{type}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg TiDB KV Request Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2161", "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:2162", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "tikv grpc avg duration", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 43 }, "hiddenSeries": false, "id": 173, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tikv_grpc_msg_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", store!=\"0\"}[1m])) by (type)/ sum(rate(tikv_grpc_msg_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", store!=\"0\"}[1m])) by (type)", "hide": false, "interval": "", "legendFormat": "{{type}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Avg TiKV GRPC Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2263", "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:2264", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "- TSO Wait Duration: The duration of a client starting to wait for the TS until received the TS result.\n- TSO RPC Duration: The duration of a client sending TSO request until received the response.\n- PD Server Handle Time: The time consumed by handling TiDB requests.", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 43 }, "hiddenSeries": false, "id": 77, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "(sum(rate(pd_client_cmd_handle_cmds_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) / sum(rate(pd_client_cmd_handle_cmds_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])))", "hide": false, "interval": "", "legendFormat": "wait - avg", "refId": "D" }, { "exemplar": true, "expr": "(sum(rate(pd_client_request_handle_requests_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) / sum(rate(pd_client_request_handle_requests_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])))", "hide": false, "interval": "", "legendFormat": "rpc - avg", "refId": "A" }, { "exemplar": true, "expr": "(sum(rate(pd_server_handle_tso_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(pd_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])))", "hide": false, "interval": "", "legendFormat": "handle - avg", "refId": "F" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"wait\"}[1m])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "wait - 99", "refId": "B" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "rpc - 99", "refId": "E" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(pd_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, le))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "handle - 99", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "PD TSO Wait/RPC Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2315", "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:2316", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "The time consumed by processing asynchronous write requests.\nStorage async write duration = store duration + apply duration", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 51 }, "hiddenSeries": false, "id": 185, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"write\"}[1m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"write\"}[1m]))", "hide": false, "interval": "", "legendFormat": "avg", "refId": "C" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", type=\"write\"}[1m])) by (le))", "hide": false, "interval": "", "legendFormat": "99", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Storage Async Write Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2371", "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:2372", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "The store time duration of each request", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 51 }, "hiddenSeries": false, "id": 183, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tikv_raftstore_store_duration_secs_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tikv_raftstore_store_duration_secs_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m]))", "hide": false, "interval": "", "legendFormat": "avg", "refId": "C" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_store_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "hide": false, "interval": "", "legendFormat": "99", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Store Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2520", "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:2521", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "The apply time duration of each request", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 51 }, "hiddenSeries": false, "id": 174, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "(sum(rate(tikv_raftstore_apply_duration_secs_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tikv_raftstore_apply_duration_secs_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])))", "hide": false, "interval": "", "legendFormat": "avg", "refId": "C" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "hide": false, "interval": "", "legendFormat": "99", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Apply Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2572", "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:2573", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "The time consumed when Raft appends log", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 9, "w": 8, "x": 0, "y": 59 }, "hiddenSeries": false, "id": 176, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "(sum(rate(tikv_raftstore_append_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tikv_raftstore_append_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])))", "hide": false, "interval": "", "legendFormat": "avg", "refId": "C" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "hide": false, "interval": "", "legendFormat": "99", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Append Log Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2672", "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:2673", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "The time consumed when Raft commits log", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 9, "w": 8, "x": 8, "y": 59 }, "hiddenSeries": false, "id": 177, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "(sum(rate(tikv_raftstore_commit_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tikv_raftstore_commit_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])))", "hide": false, "interval": "", "legendFormat": "avg", "refId": "C" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_commit_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "hide": false, "interval": "", "legendFormat": "99", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Commit Log Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2724", "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:2725", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "The time consumed for Raft to apply logs", "editable": true, "error": false, "fill": 1, "fillGradient": 0, "grid": {}, "gridPos": { "h": 9, "w": 8, "x": 16, "y": 59 }, "hiddenSeries": false, "id": 186, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "(sum(rate(tikv_raftstore_apply_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tikv_raftstore_apply_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])))", "hide": false, "interval": "", "legendFormat": "avg", "refId": "C" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "hide": false, "interval": "", "legendFormat": "99", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Apply Log Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2776", "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:2777", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": true, "datasource": "${DS_TEST-CLUSTER}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 68 }, "id": 159, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The CPU usage of each TiFlash instance", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 2 }, "hiddenSeries": false, "id": 161, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"tiflash\"}[1m])", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The memory usage per TiFlash instance", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 2 }, "hiddenSeries": false, "id": 1709, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "avg(tiflash_proxy_process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance)", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "IO Throughput per TiFlash instance", "editable": true, "error": false, "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 2 }, "hiddenSeries": false, "id": 165, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(node_disk_read_bytes_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"tiflash.*\"}[1m])) by (instance)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "read - {{instance}} ", "refId": "A", "step": 10 }, { "exemplar": true, "expr": "sum(rate(node_disk_written_bytes_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"tiflash.*\"}[1m])) by (instance)", "hide": false, "interval": "", "legendFormat": "write - {{instance}} ", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "IO Throughput", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:250", "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:251", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The MPP query count in TiFlash", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 157, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:608", "alias": "Failed", "color": "#C4162A", "linewidth": 2, "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "max(tiflash_mpp_task_manager{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "refId": "A" }, { "exemplar": true, "expr": "sum(rate(tiflash_coprocessor_request_error{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m]))", "hide": false, "interval": "", "legendFormat": "Failed", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "MPP Query count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of coprocessor requests received by all TiFlash instances. batch is the number of batch requests. batch_cop is the number of coprocessor requests in the batch requests. cop is the number of coprocessor requests that are sent directly via the coprocessor interface. cop_dag is the number of dag requests in all coprocessor requests. super_batch is the number of requests to enable the Super Batch feature.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 9 }, "hiddenSeries": false, "id": 9, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tiflash_coprocessor_request_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request QPS", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of each type of dag executors in the requests received by all TiFlash instances. table_scan is the table scan executor. selection is the selection executor. aggregation is the aggregation executor. top_n is the TopN executor. limit is the limit executor.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 9 }, "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tiflash_coprocessor_executor_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Executor QPS", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": true, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The overview of the total duration of all TiFlash instances processing coprocessor requests.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 0, "y": 16 }, "hiddenSeries": false, "id": 166, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": true, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tiflash_coprocessor_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request Duration Overview", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": " The total duration of all TiFlash instances processing coprocessor requests.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 8, "y": 16 }, "hiddenSeries": false, "id": 11, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, "legendFormat": "999", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,type))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "99-{{type}}", "refId": "B" }, { "exemplar": true, "expr": "sum(rate(tiflash_coprocessor_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type) /sum(rate(tiflash_coprocessor_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}}", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of all TiFlash instances processing coprocessor requests. The processing time is from starting to execute the coprocessor request to completing the execution.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 8, "x": 16, "y": 16 }, "hiddenSeries": false, "id": 13, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, "legendFormat": "999", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le,type))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "99-{{type}}", "refId": "B" }, { "exemplar": true, "expr": "sum(rate(tiflash_coprocessor_request_handle_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type) /sum(rate(tiflash_coprocessor_request_handle_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}}", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Request Handle Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 22 }, "hiddenSeries": false, "id": 23763571997, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.999, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (le, type))", "interval": "", "legendFormat": "999-{{type}}", "queryType": "randomWalk", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (le, type))", "hide": false, "interval": "", "legendFormat": "99-{{type}}", "queryType": "randomWalk", "refId": "B" }, { "exemplar": true, "expr": "histogram_quantile(0.80, sum(rate(tiflash_coprocessor_request_memory_usage_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (le, type))", "hide": false, "interval": "", "legendFormat": "80-{{type}}", "queryType": "randomWalk", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Cop task memory usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:642", "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:643", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 22 }, "hiddenSeries": false, "id": 23763571999, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tiflash_coprocessor_response_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[2m])) by (type)", "interval": "", "legendFormat": "{{type}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Response Bytes/Seconds", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:710", "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:711", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The time used by wait_index for all TiFlash instances, namely the time used to wait until local index >= read_index after the read_index request is received.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 37, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:538", "alias": "/timeout/", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, "legendFormat": "max", "refId": "A" }, { "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "99", "refId": "B" }, { "exemplar": true, "expr": "sum(rate(tiflash_raft_wait_index_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))/sum(rate(tiflash_raft_wait_index_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "avg", "refId": "D" }, { "expr": "sum(increase(tiflash_system_profile_event_RaftWaitIndexTimeout{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-timeout", "refId": "E" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Raft Wait Index Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": 2, "format": "opm", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of times that each TiFlash instance triggers the read_index request per second, which equals to the number of Regions triggered.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 12, "x": 12, "y": 29 }, "hiddenSeries": false, "id": 23763571994, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "/timeout/", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(1.00, sum(rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, "legendFormat": "max", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "99", "refId": "B" }, { "exemplar": true, "expr": "sum(rate(tiflash_raft_read_index_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))/sum(rate(tiflash_raft_read_index_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "avg", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Raft Batch Read Index Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": 2, "format": "opm", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The throughput of write by instance", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 0, "fillGradient": 0, "gridPos": { "h": 6, "w": 24, "x": 0, "y": 35 }, "height": "", "hiddenSeries": false, "id": 89, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "/total/", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"write\"}[1m])) by (instance)", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "write-{{instance}}", "refId": "A", "step": 10 }, { "exemplar": true, "expr": "sum(rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"ingest\"}[1m])) by (instance)", "hide": false, "interval": "", "legendFormat": "ingest-{{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Write Throughput By Instance", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The flow of different kinds of write operations", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 41 }, "height": "", "hiddenSeries": false, "id": 60, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "File Descriptor", "refId": "A", "step": 10 }, { "expr": "sum(rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Page", "refId": "B" }, { "expr": "sum(rate(tiflash_system_profile_event_PSMBackgroundWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "PageBackGround", "refId": "C" }, { "expr": "sum(rate(tiflash_system_profile_event_WriteBufferAIOWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 1, "legendFormat": "AIO", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Write flow", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The flow of different kinds of read operations", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 41 }, "height": "", "hiddenSeries": false, "id": 59, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 2, "legendFormat": "File Descriptor", "refId": "A", "step": 10 }, { "expr": "sum(rate(tiflash_system_profile_event_PSMReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "Page", "refId": "B" }, { "expr": "sum(rate(tiflash_system_profile_event_PSMBackgroundReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "PageBackGround", "refId": "C" }, { "expr": "sum(rate(tiflash_system_profile_event_ReadBufferAIOReadBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m]))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "AIO", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read flow", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "TiFlash", "type": "row" }, { "collapsed": true, "datasource": "${DS_TEST-CLUSTER}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 69 }, "id": 515, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "CPU usage of TiCDC", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 71 }, "hiddenSeries": false, "id": 24, "legend": { "alignAsTable": true, "avg": true, "current": true, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sort": "current", "sortDesc": false, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "paceLength": 10, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [ { "$$hashKey": "object:284", "alias": "/.*MaxProcs/", "fill": 0, "linewidth": 2, "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}[1m])", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", "refId": "A" }, { "expr": "ticdc_server_go_max_procs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-MaxProcs", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "CPU usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:295", "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:296", "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Memory usage of TiCDC", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 71 }, "hiddenSeries": false, "id": 23, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "paceLength": 10, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "process-{{instance}}", "refId": "A" }, { "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "heap-{{instance}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Memory usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Goroutine count of TiCDC", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 71 }, "hiddenSeries": false, "id": 26, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "paceLength": 10, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": " go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}", "refId": "A" }, { "expr": "go_threads{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=\"ticdc\"}", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "threads-{{instance}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Goroutine count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The lag between changefeed checkpoint ts and the latest ts of upstream TiDB.", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 78 }, "hiddenSeries": false, "id": 3, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "paceLength": 10, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "max(ticdc_owner_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (changefeed)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{changefeed}}", "refId": "A" }, { "exemplar": true, "expr": "max(ticdc_processor_checkpoint_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance,changefeed)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{changefeed}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Changefeed checkpoint lag", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:79", "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:80", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The lag between changefeed resolved ts and the latest ts of upstream TiDB.", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 78 }, "hiddenSeries": false, "id": 513, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "paceLength": 10, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "max(ticdc_owner_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (changefeed)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{changefeed}}", "refId": "C" }, { "exemplar": true, "expr": "max(ticdc_processor_resolved_ts_lag{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (instance,changefeed)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{chanefeed}}", "refId": "D" } ], "thresholds": [], "timeRegions": [], "title": "Changefeed resolved ts lag", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The status of each changefeed.\n\n0: Normal\n\n1: Error\n\n2: Failed\n\n3: Stopped\n\n4: Finished\n\n-1: Unknown", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 78 }, "hiddenSeries": false, "id": 163, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 1, "points": true, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "max(ticdc_owner_status{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}) by (changefeed)", "format": "time_series", "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{changefeed}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "The status of changefeeds", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of events that puller outputs to sorter \n per second", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 85 }, "hiddenSeries": false, "id": 218, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "paceLength": 10, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum (rate(ticdc_puller_txn_collect_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type!=\"resolved\"}[1m])) by (changefeed, instance, type)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{instance}}-{{type}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Puller output events/s", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of events that sorter outputs to puller \n per second", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 85 }, "hiddenSeries": false, "id": 228, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "paceLength": 10, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(ticdc_sorter_output_event_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (changefeed, instance, type)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{instance}}-{{type}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Sorter output events/s", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of events that mounter outputs to sink per second", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 92 }, "hiddenSeries": false, "id": 219, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "paceLength": 10, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(ticdc_mounter_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (instance,changefeed)", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{instance}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Mounter output events/s", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:196", "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:197", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of events that table sorter outputs to sink per second", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 92 }, "hiddenSeries": false, "id": 108, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(ticdc_sink_table_sink_total_rows_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (changefeed, instance)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{instance}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Table sink output events/s", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of rows that sink flushes to downstream per second.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 99 }, "hiddenSeries": false, "id": 654, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(ticdc_sinkv2_batch_row_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (changefeed, instance)", "interval": "", "legendFormat": "{{changefeed}}-{{instance}}", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "SinkV2 - Sink flush rows/s", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:258", "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:259", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Full flush (backend flush + callback + conflict detector notify) duration", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 99 }, "hiddenSeries": false, "id": 620, "legend": { "alignAsTable": true, "avg": true, "current": false, "max": true, "min": false, "rightSide": false, "show": true, "sort": "avg", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.999, sum(rate(ticdc_sinkv2_txn_worker_flush_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (le,changefeed,instance))", "interval": "", "legendFormat": "99.9-{{changefeed}}-{{instance}}", "queryType": "randomWalk", "refId": "A" }, { "exemplar": true, "expr": "sum(rate(ticdc_sinkv2_txn_worker_flush_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (changefeed,instance) / \nsum(rate(ticdc_sinkv2_txn_worker_flush_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (changefeed,instance)", "hide": false, "interval": "", "legendFormat": "avg-{{changefeed}}-{{instance}}", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "Transaction Sink Full Flush Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:332", "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "$$hashKey": "object:333", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "MQ worker send messages to Kafka, this metric record the time cost on send every message.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 106 }, "hiddenSeries": false, "id": 653, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.999, sum(rate(ticdc_sinkv2_mq_worker_send_message_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (le,changefeed,instance))", "interval": "", "legendFormat": "{{changefeed}}-{{instance}}-P999", "queryType": "randomWalk", "refId": "A" }, { "exemplar": true, "expr": "sum(rate(ticdc_sinkv2_mq_worker_send_message_duration_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (changefeed,instance) / \nsum(rate(ticdc_sinkv2_mq_worker_send_message_duration_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}[1m])) by (changefeed,instance)", "hide": false, "interval": "", "legendFormat": "{{changefeed}}-{{instance}}-avg", "refId": "B" } ], "thresholds": [], "timeRegions": [], "title": "MQ Worker Send Message Duration Percentile", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:406", "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "$$hashKey": "object:407", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Bytes/second written off all brokers.\nvalue = one-minute moving average rate of Bytes per second", "fieldConfig": { "defaults": { "links": [] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 106 }, "hiddenSeries": false, "id": 628, "legend": { "alignAsTable": true, "avg": true, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "paceLength": 10, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(ticdc_sinkv2_kafka_producer_outgoing_byte_rate{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", changefeed!=\"\"}) by (changefeed, instance, broker)", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "{{changefeed}}-{{instance}}-{{broker}}", "refId": "A" } ], "thresholds": [], "timeRegions": [], "title": "Kafka Outgoing Bytes", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:480", "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:481", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "CDC", "type": "row" } ], "refresh": "30s", "schemaVersion": 27, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": { "isNone": true, "selected": false, "text": "None", "value": "" }, "datasource": "${DS_TEST-CLUSTER}", "definition": "", "description": null, "error": null, "hide": 2, "includeAll": false, "label": "K8s-cluster", "multi": false, "name": "k8s_cluster", "options": [], "query": { "query": "label_values(pd_cluster_status, k8s_cluster)", "refId": "${DS_TEST-CLUSTER}-k8s_cluster-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { "isNone": true, "selected": false, "text": "None", "value": "" }, "datasource": "${DS_TEST-CLUSTER}", "definition": "", "description": null, "error": null, "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", "options": [], "query": { "query": "label_values(pd_cluster_status{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refId": "${DS_TEST-CLUSTER}-tidb_cluster-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Test-Cluster-Performance-Overview", "uid": "eDbRZpnWa", "version": 2 }