{ "uid": "shared-resource-diagnosis", "title": "[SharedScope]Shared Resource One Page", "tags": [ "cluster", "shared-resource", "resource-control", "tidb", "tikv" ], "timezone": "browser", "schemaVersion": 41, "version": 1, "refresh": "1m", "time": { "from": "now-1h", "to": "now" }, "templating": { "list": [ { "name": "k8s_cluster", "label": "K8s cluster", "type": "query", "hide": 2, "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(tikv_engine_size_bytes, k8s_cluster)", "query": { "query": "label_values(tikv_engine_size_bytes, k8s_cluster)", "refId": "StandardVariableQuery" }, "includeAll": false }, { "name": "cluster_id", "label": "Logical Cluster", "type": "query", "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(tidb_server_query_total{sharedpool_id=\"$tidb_cluster\"}, tidb_cluster_id)", "query": { "query": "label_values(tidb_server_query_total{sharedpool_id=\"$tidb_cluster\"}, tidb_cluster_id)", "refId": "StandardVariableQuery" }, "includeAll": true, "allValue": ".+", "multi": true, "current": { "text": "All", "value": [ "$__all" ] } }, { "name": "keyspace_id", "label": "Keyspace ID", "type": "query", "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values({__name__=~\"tikv_cpu_token_wait_duration_seconds_count|tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_count\",k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}, keyspace_id)", "query": { "query": "label_values({__name__=~\"tikv_cpu_token_wait_duration_seconds_count|tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_count\",k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}, keyspace_id)", "refId": "StandardVariableQuery" }, "includeAll": true, "allValue": ".*", "multi": true, "current": { "text": "All", "value": [ "$__all" ] }, "sort": 1 }, { "name": "tikv_instance", "label": "TiKV instance", "type": "query", "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}, instance)", "query": { "query": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}, instance)", "refId": "StandardVariableQuery" }, "includeAll": true, "allValue": ".*", "multi": true, "current": { "text": "All", "value": [ "$__all" ] }, "sort": 1 }, { "name": "shared_resource_scope", "label": "Shared resource scope", "type": "constant", "hide": 2, "query": "resource_pool", "current": { "selected": true, "text": "resource_pool", "value": "resource_pool" }, "options": [ { "selected": true, "text": "resource_pool", "value": "resource_pool" } ], "skipUrlSync": false } ] }, "panels": [ { "cacheTimeout": null, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "height": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "hideTimeOverride": false, "id": 1, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "TiDB By ClusterID", "transparent": false, "transformations": [], "collapsed": false, "panels": [], "type": "row" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": "" }, "color": null, "custom": {}, "decimals": null, "mappings": null, "unit": "short", "noValue": "none" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 4, "x": 0, "y": 1 }, "hideTimeOverride": false, "id": 2, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "count(count by (tidb_cluster_id) (tidb_server_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}))", "query": "count(count by (tidb_cluster_id) (tidb_server_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Clusters", "transparent": false, "transformations": [], "options": { "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": "" }, "color": null, "custom": {}, "decimals": null, "mappings": null, "unit": "ops", "noValue": "none" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 4, "x": 4, "y": 1 }, "hideTimeOverride": false, "id": 3, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "sum(rate(tidb_server_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\",result=\"OK\"}[$__rate_interval]))", "query": "sum(rate(tidb_server_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\",result=\"OK\"}[$__rate_interval]))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "QPS", "transparent": false, "transformations": [], "options": { "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": "" }, "color": null, "custom": {}, "decimals": null, "mappings": null, "unit": "s", "noValue": "none" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 4, "x": 8, "y": 1 }, "hideTimeOverride": false, "id": 4, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])) by (le))", "query": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])) by (le))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "P99 latency", "transparent": false, "transformations": [], "options": { "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": "" }, "color": null, "custom": {}, "decimals": null, "mappings": null, "unit": "ops", "noValue": "none" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 4, "x": 12, "y": 1 }, "hideTimeOverride": false, "id": 5, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "sum(rate(tidb_server_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\",result=\"Error\"}[$__rate_interval]))", "query": "sum(rate(tidb_server_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\",result=\"Error\"}[$__rate_interval]))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Error QPS", "transparent": false, "transformations": [], "options": { "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": "" }, "color": null, "custom": {}, "decimals": null, "mappings": null, "unit": "ops", "noValue": "none" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 4, "x": 16, "y": 1 }, "hideTimeOverride": false, "id": 6, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "sum(rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval])) + sum(rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval]))", "query": "sum(rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval])) + sum(rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval]))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "RU / s", "transparent": false, "transformations": [], "options": { "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": "" }, "color": null, "custom": {}, "decimals": null, "mappings": null, "unit": "short", "noValue": "none" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 4, "x": 20, "y": 1 }, "hideTimeOverride": false, "id": 7, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "sum(tidb_server_connections{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"})", "query": "sum(tidb_server_connections{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"})", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Connections", "transparent": false, "transformations": [], "options": { "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "TiDB process CPU divided by its GOMAXPROCS quota for each logical cluster. 1.0 is displayed as 100%.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 } ] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "percentunit" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 8 }, "hideTimeOverride": false, "id": 8, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, sum by (tidb_cluster_id) (rate(process_cpu_seconds_total{sharedpool_id=\"$tidb_cluster\",component=~\"worker-tidb|tidb\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])) / sum by (tidb_cluster_id) (tidb_server_maxprocs{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}))", "query": "topk(10, sum by (tidb_cluster_id) (rate(process_cpu_seconds_total{sharedpool_id=\"$tidb_cluster\",component=~\"worker-tidb|tidb\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])) / sum by (tidb_cluster_id) (tidb_server_maxprocs{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "cluster-{{tidb_cluster_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "TiDB Server CPU utilization by cluster", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Healthy and unavailable TiDB Server scrape targets by logical cluster. Any down series above zero requires investigation.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "short" }, "overrides": [ { "matcher": { "id": "byRegexp", "options": "/^up ·/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } } ] }, { "matcher": { "id": "byRegexp", "options": "/^down ·/" }, "properties": [ { "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } } ] } ] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 8 }, "hideTimeOverride": false, "id": 9, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, sum by (tidb_cluster_id) (up{sharedpool_id=\"$tidb_cluster\",component=~\"worker-tidb|tidb\",tidb_cluster_id=~\"$cluster_id\"}))", "query": "topk(10, sum by (tidb_cluster_id) (up{sharedpool_id=\"$tidb_cluster\",component=~\"worker-tidb|tidb\",tidb_cluster_id=~\"$cluster_id\"}))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "up · cluster-{{tidb_cluster_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" }, { "expr": "topk(10, count by (tidb_cluster_id) (up{sharedpool_id=\"$tidb_cluster\",component=~\"worker-tidb|tidb\",tidb_cluster_id=~\"$cluster_id\"}) - sum by (tidb_cluster_id) (up{sharedpool_id=\"$tidb_cluster\",component=~\"worker-tidb|tidb\",tidb_cluster_id=~\"$cluster_id\"}))", "query": "topk(10, count by (tidb_cluster_id) (up{sharedpool_id=\"$tidb_cluster\",component=~\"worker-tidb|tidb\",tidb_cluster_id=~\"$cluster_id\"}) - sum by (tidb_cluster_id) (up{sharedpool_id=\"$tidb_cluster\",component=~\"worker-tidb|tidb\",tidb_cluster_id=~\"$cluster_id\"}))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "down · cluster-{{tidb_cluster_id}}", "metric": "", "refId": "B", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "TiDB Server status by cluster", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Successful SQL throughput by logical cluster. All-cluster views are capped at Top 10; select a cluster for exact drill-down.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "ops" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 15 }, "hideTimeOverride": false, "id": 10, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, sum by (tidb_cluster_id) (rate(tidb_server_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\",result=\"OK\"}[$__rate_interval])))", "query": "topk(10, sum by (tidb_cluster_id) (rate(tidb_server_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\",result=\"OK\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "cluster-{{tidb_cluster_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "QPS by cluster", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Operational thresholds: green below 100 ms, yellow below 500 ms, red at or above 500 ms. Tune to the cluster SLA when one is defined.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.5 } ] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "s" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 15 }, "hideTimeOverride": false, "id": 11, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])) by (tidb_cluster_id, le)))", "query": "topk(10, histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])) by (tidb_cluster_id, le)))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "P99 · cluster-{{tidb_cluster_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" }, { "expr": "topk(10, histogram_quantile(0.90, sum(rate(tidb_server_handle_query_duration_seconds_bucket{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])) by (tidb_cluster_id, le)))", "query": "topk(10, histogram_quantile(0.90, sum(rate(tidb_server_handle_query_duration_seconds_bucket{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])) by (tidb_cluster_id, le)))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "P90 · cluster-{{tidb_cluster_id}}", "metric": "", "refId": "B", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Latency by cluster", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Failed SQL throughput by logical cluster, capped at Top 10 for the all-cluster view.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "bars", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 0, "fillOpacity": 80, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "normal", "group": "A" }, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "ops" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 22 }, "hideTimeOverride": false, "id": 12, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, sum by (tidb_cluster_id) (rate(tidb_server_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\",result=\"Error\"}[$__rate_interval])))", "query": "topk(10, sum by (tidb_cluster_id) (rate(tidb_server_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\",result=\"Error\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "cluster-{{tidb_cluster_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Error QPS by cluster", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "New slow queries per second by logical cluster, capped at Top 10 for the all-cluster view.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "bars", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 0, "fillOpacity": 80, "gradientMode": "none", "spanNulls": false, "showPoints": "never", "pointSize": 5, "stacking": { "mode": "normal", "group": "A" }, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "ops" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 22 }, "hideTimeOverride": false, "id": 13, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, sum by (tidb_cluster_id) (rate(tidb_server_slow_query_process_duration_seconds_count{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])))", "query": "topk(10, sum by (tidb_cluster_id) (rate(tidb_server_slow_query_process_duration_seconds_count{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "cluster-{{tidb_cluster_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Slow queries by cluster", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rows changed by each logical cluster across SQL types, capped at Top 10.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "rows/s" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 29 }, "hideTimeOverride": false, "id": 14, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, sum by (tidb_cluster_id) (rate(tidb_executor_affected_rows{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])))", "query": "topk(10, sum by (tidb_cluster_id) (rate(tidb_executor_affected_rows{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "cluster-{{tidb_cluster_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Affected rows by cluster", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "P99 connection idle duration while a transaction remains open, by logical cluster.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.5 } ] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "s" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 29 }, "hideTimeOverride": false, "id": 15, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, histogram_quantile(0.99, sum by (tidb_cluster_id, le) (rate(tidb_server_conn_idle_duration_seconds_bucket{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\",in_txn=\"1\"}[$__rate_interval]))))", "query": "topk(10, histogram_quantile(0.99, sum by (tidb_cluster_id, le) (rate(tidb_server_conn_idle_duration_seconds_bucket{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\",in_txn=\"1\"}[$__rate_interval]))))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "cluster-{{tidb_cluster_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Idle in transaction P99", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Read and write RU consumption rate by logical cluster in the shared PD resource manager.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "ops" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 36 }, "hideTimeOverride": false, "id": 16, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, sum by (keyspace_name) (rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval])))", "query": "topk(10, sum by (keyspace_name) (rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "read · keyspace-{{keyspace_name}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" }, { "expr": "topk(10, sum by (keyspace_name) (rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval])))", "query": "topk(10, sum by (keyspace_name) (rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "write · keyspace-{{keyspace_name}}", "metric": "", "refId": "B", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "RU read / write by cluster", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "RU efficiency by logical cluster. A rise while QPS is stable points to a more expensive query mix.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "short" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 36 }, "hideTimeOverride": false, "id": 17, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, (sum by (keyspace_name) (rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval]) + rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval]))) / on (keyspace_name) sum by (keyspace_name) (rate(tidb_session_resource_group_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])))", "query": "topk(10, (sum by (keyspace_name) (rate(resource_manager_resource_unit_read_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval]) + rate(resource_manager_resource_unit_write_request_unit_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",keyspace_name=~\"$cluster_id\",type=\"tp\"}[$__rate_interval]))) / on (keyspace_name) sum by (keyspace_name) (rate(tidb_session_resource_group_query_total{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "keyspace-{{keyspace_name}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "RU / query by cluster", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Logical coprocessor response bytes returned from TiKV to TiDB by cluster. This is request-level logical throughput, not physical disk IO.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "Bps" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 36 }, "hideTimeOverride": false, "id": 18, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, sum by (tidb_cluster_id) (rate(tidb_ruv2_tikv_coprocessor_response_bytes{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])))", "query": "topk(10, sum by (tidb_cluster_id) (rate(tidb_ruv2_tikv_coprocessor_response_bytes{sharedpool_id=\"$tidb_cluster\",tidb_cluster_id=~\"$cluster_id\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "cluster-{{tidb_cluster_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Cop response throughput by cluster", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "height": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 43 }, "hideTimeOverride": false, "id": 19, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "TiKV Overview", "transparent": false, "transformations": [], "collapsed": false, "panels": [], "type": "row" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Whole-process CPU utilisation against TiKV's configured CPU-core quota, by instance.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.7 }, { "color": "red", "value": 0.9 } ] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "percentunit" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 44 }, "hideTimeOverride": false, "id": 20, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "sum by (instance) (rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval])) / on (instance) group_left sum by (instance) (tikv_server_cpu_cores_quota{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"})", "query": "sum by (instance) (rate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval])) / on (instance) group_left sum by (instance) (tikv_server_cpu_cores_quota{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"})", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "TiKV CPU", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "P99 execution latency of TiKV gRPC message types, matching TiKV Details gRPC message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.5 } ] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "s" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 44 }, "hideTimeOverride": false, "id": 21, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(12, histogram_quantile(0.99, sum by (type, le) (rate(tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",type!=\"kv_gc\"}[$__rate_interval]))))", "query": "topk(12, histogram_quantile(0.99, sum by (type, le) (rate(tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",type!=\"kv_gc\"}[$__rate_interval]))))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "TiKV gRPC latency P99", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "TiKV engine IO throughput split by operation and workload source. Top 10 active contributors only.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "Bps" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 51 }, "hideTimeOverride": false, "id": 22, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, sum by (op, type) (rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval])) > 0)", "query": "topk(10, sum by (op, type) (rate(tikv_io_bytes{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval])) > 0)", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{op}} · {{type}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "IO throughput breakdown", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "P99 storage request and write-path waiting latency. Use it with IO throughput to identify storage saturation and write backpressure.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.5 } ] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "s" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 51 }, "hideTimeOverride": false, "id": 23, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "histogram_quantile(0.99, sum by (type, le) (rate(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",type=~\"snapshot_read_index_propose_wait|snapshot_read_index_confirm|write\"}[$__rate_interval])))", "query": "histogram_quantile(0.99, sum by (type, le) (rate(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",type=~\"snapshot_read_index_propose_wait|snapshot_read_index_confirm|write\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "storage · {{type}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" }, { "expr": "histogram_quantile(0.99, sum by (le) (rate(tikv_raftstore_store_write_task_wait_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval])))", "query": "histogram_quantile(0.99, sum by (le) (rate(tikv_raftstore_store_write_task_wait_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "write task wait", "metric": "", "refId": "B", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" }, { "expr": "histogram_quantile(0.99, sum by (le) (rate(tikv_raftstore_store_min_write_pause_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval])))", "query": "histogram_quantile(0.99, sum by (le) (rate(tikv_raftstore_store_min_write_pause_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "minimum write pause", "metric": "", "refId": "C", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Storage & IO wait P99", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Storage-side queueing by TiKV YATP pool. Use this to identify CPU/thread-pool saturation in the shared physical pool.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.5 } ] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "s" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 58 }, "hideTimeOverride": false, "id": 24, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "(histogram_quantile(0.99, sum by (name, le) (rate(tikv_yatp_pool_schedule_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval]))) or vector(0))", "query": "(histogram_quantile(0.99, sum by (name, le) (rate(tikv_yatp_pool_schedule_wait_duration_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval]))) or vector(0))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Thread pool wait P99", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Wait introduced by TiKV rate limiting, grouped by priority class. This is a shared-pool throttling signal.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.5 } ] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "s" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 58 }, "hideTimeOverride": false, "id": 25, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "(histogram_quantile(0.99, sum by (type, le) (rate(tikv_rate_limiter_request_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval]))) or vector(0))", "query": "(histogram_quantile(0.99, sum by (type, le) (rate(tikv_rate_limiter_request_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval]))) or vector(0))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Rate-limit wait P99", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "height": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 65 }, "hideTimeOverride": false, "id": 26, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "TiKV · Unified Read Pool & Fairness", "transparent": false, "transformations": [], "collapsed": false, "panels": [], "type": "row" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": "" }, "color": null, "custom": {}, "decimals": null, "mappings": null, "unit": "cores", "noValue": "none" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 66 }, "hideTimeOverride": false, "id": 27, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "sum(rate(tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval]))", "query": "sum(rate(tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval]))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Read CPU cores", "transparent": false, "transformations": [], "options": { "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": "" }, "color": null, "custom": {}, "decimals": null, "mappings": null, "unit": "short", "noValue": "none" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 66 }, "hideTimeOverride": false, "id": 28, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "count(sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) > 0)", "query": "count(sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) > 0)", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Active keyspaces", "transparent": false, "transformations": [], "options": { "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": "" }, "color": null, "custom": {}, "decimals": null, "mappings": null, "unit": "s", "noValue": "none" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 66 }, "hideTimeOverride": false, "id": 29, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "histogram_quantile(0.99, sum by (le) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])))", "query": "histogram_quantile(0.99, sum by (le) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Slice wait P99", "transparent": false, "transformations": [], "options": { "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": "" }, "color": null, "custom": {}, "decimals": null, "mappings": null, "unit": "short", "noValue": "none" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 66 }, "hideTimeOverride": false, "id": 30, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "max(max by (instance) (tikv_yatp_pool_keyspace_fair_queue_group_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\"}))", "query": "max(max by (instance) (tikv_yatp_pool_keyspace_fair_queue_group_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\"}))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Max groups / TiKV", "transparent": false, "transformations": [], "options": { "textMode": "auto", "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "horizontal", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false } }, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Instantaneous tasks executing in each TiKV unified read pool. This identifies which physical store is busy; it cannot yet attribute CPU to a logical cluster or SQL pattern.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "short" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 73 }, "hideTimeOverride": false, "id": 31, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "sum by (instance) (tikv_unified_read_pool_running_tasks{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"})", "query": "sum by (instance) (tikv_unified_read_pool_running_tasks{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"})", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Running tasks", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Request-class queueing in TiKV. This is the current best signal for anomalous internal work (for example analyse/checksum-style classes), but the req label does not establish logical-cluster ownership.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.5 } ] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "s" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 80 }, "hideTimeOverride": false, "id": 32, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(12, histogram_quantile(0.99, sum by (req, le) (rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval]))) )", "query": "topk(12, histogram_quantile(0.99, sum by (req, le) (rate(tikv_coprocessor_request_wait_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval]))) )", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Coprocessor queue P99", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Execution latency by bounded TiKV request class. Pair it with the queue-wait panel to distinguish CPU/pool contention from expensive execution.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.5 } ] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "s" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 80 }, "hideTimeOverride": false, "id": 33, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(12, histogram_quantile(0.99, sum by (req, le) (rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval]))) )", "query": "topk(12, histogram_quantile(0.99, sum by (req, le) (rate(tikv_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\"}[$__rate_interval]))) )", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Coprocessor exec P99", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "#5534 CPU and wall time rate for selected unified-read keyspaces. All-keyspace views are capped at Top 10 to control series cardinality.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "short" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 87 }, "hideTimeOverride": false, "id": 34, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])))", "query": "topk(10, sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "CPU · KSID-{{keyspace_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" }, { "expr": "topk(10, sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_task_wall_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])))", "query": "topk(10, sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_task_wall_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "wall · KSID-{{keyspace_id}}", "metric": "", "refId": "B", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "CPU / wall · Top 10", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "#5534 P99 and average time from enqueueing a task slice until the fair queue pops it.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "yellow", "value": 0.1 }, { "color": "red", "value": 0.5 } ] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "s" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 87 }, "hideTimeOverride": false, "id": 35, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, histogram_quantile(0.99, sum by (keyspace_id, le) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval]))))", "query": "topk(10, histogram_quantile(0.99, sum by (keyspace_id, le) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval]))))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "P99 · KSID-{{keyspace_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" }, { "expr": "topk(10, sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) / sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])))", "query": "topk(10, sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) / sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg · KSID-{{keyspace_id}}", "metric": "", "refId": "B", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Slice wait · P99 / avg", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "#5534 P99 and average queue depth observed when a task slice enters the fair queue.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "short" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 94 }, "hideTimeOverride": false, "id": 36, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "topk(10, histogram_quantile(0.99, sum by (keyspace_id, le) (rate(tikv_yatp_pool_keyspace_fair_queue_length_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval]))))", "query": "topk(10, histogram_quantile(0.99, sum by (keyspace_id, le) (rate(tikv_yatp_pool_keyspace_fair_queue_length_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval]))))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "P99 · KSID-{{keyspace_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" }, { "expr": "topk(10, sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_length_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) / sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_length_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])))", "query": "topk(10, sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_length_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) / sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_length_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])))", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg · KSID-{{keyspace_id}}", "metric": "", "refId": "B", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Queue depth · P99 / avg", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "#5534 distribution of other successful task-slice pops seen while selected keyspace slices wait. Multiple selected keyspaces are aggregated.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 94 }, "hideTimeOverride": false, "id": 37, "interval": null, "links": [], "maxDataPoints": 512, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "sum(increase(tikv_yatp_pool_keyspace_fair_queue_task_slice_waits_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) by (le)", "query": "sum(increase(tikv_yatp_pool_keyspace_fair_queue_task_slice_waits_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) by (le)", "target": "", "format": "heatmap", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Task slice waits · distribution", "transparent": false, "transformations": [], "cards": { "cardPadding": null, "cardRound": null }, "color": { "mode": "spectrum", "cardColor": "#b4ff00", "colorScale": "sqrt", "exponent": 0.5, "colorScheme": "interpolateSpectral", "max": null, "min": null }, "dataFormat": "tsbuckets", "heatmap": {}, "hideZeroBuckets": true, "highlightCards": true, "legend": { "show": false }, "options": {}, "reverseYBuckets": false, "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual", "showHistogram": true, "show": true }, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "values": [], "show": true }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "#5534 average CPU and wall time consumed by each popped unified-read task slice.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "ms" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 101 }, "hideTimeOverride": false, "id": 38, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "(sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) / sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval]))) * 1000", "query": "(sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) / sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval]))) * 1000", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "CPU · KSID-{{keyspace_id}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" }, { "expr": "(sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_task_wall_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) / sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval]))) * 1000", "query": "(sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_task_wall_seconds_total{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval])) / sum by (keyspace_id) (rate(tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}[$__rate_interval]))) * 1000", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "wall · KSID-{{keyspace_id}}", "metric": "", "refId": "B", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "CPU / wall per slice", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "bottom", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "#5534 active keyspace groups in each TiKV unified-read fair queue. This is a scheduling-scale guardrail, not CPU usage.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "color": { "mode": "palette-classic" }, "custom": { "axisPlacement": "auto", "axisLabel": "", "drawStyle": "line", "lineInterpolation": "linear", "barAlignment": 0, "lineWidth": 1, "fillOpacity": 0, "gradientMode": "none", "spanNulls": false, "showPoints": "auto", "pointSize": 5, "stacking": {}, "scaleDistribution": { "type": "linear", "log": 2 }, "hideFrom": { "tooltip": false, "viz": false, "legend": false }, "thresholdsStyle": { "mode": "off" }, "axisSoftMin": null, "axisSoftMax": null }, "mappings": [], "min": null, "max": null, "decimals": null, "unit": "short" }, "overrides": [] }, "height": null, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 101 }, "hideTimeOverride": false, "id": 39, "interval": null, "links": [], "maxDataPoints": 100, "minSpan": null, "repeat": null, "repeatDirection": null, "maxPerRow": null, "span": null, "targets": [ { "expr": "max by (instance) (tikv_yatp_pool_keyspace_fair_queue_group_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\"})", "query": "max by (instance) (tikv_yatp_pool_keyspace_fair_queue_group_count{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$tikv_instance\",name=~\"unified-read.*\"})", "target": "", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "refId": "A", "step": 10, "instant": false, "datasource": "${DS_TEST-CLUSTER}" } ], "timeFrom": null, "timeShift": null, "title": "Active groups by TiKV", "transparent": false, "transformations": [], "options": { "legend": { "displayMode": "list", "placement": "right", "calcs": [] }, "tooltip": { "mode": "multi", "sort": "none" } }, "type": "timeseries" } ], "__inputs": [ { "name": "DS_TEST-CLUSTER", "label": "Prometheus", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ] }