{ "__inputs": [ { "name": "DS_TEST-CLUSTER", "label": "test-cluster", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "6.1.6" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "${DS_TEST-CLUSTER}", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "id": null, "links": [], "panels": [ { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 33, "panels": [ { "alert": { "conditions": [ { "evaluator": { "params": [ 0.5 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Duration alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "TiDB query durations by histogram buckets with different percents", "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 1 }, "id": 3, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "999", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.99, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 3, "legendFormat": "99", "refId": "B", "step": 15 }, { "expr": "histogram_quantile(0.95, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95", "refId": "C" }, { "expr": "histogram_quantile(0.80, sum(rate(tidb_server_handle_query_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80", "refId": "D" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.5 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 2 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Get Token Duration alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Duration (us) for getting token, it should be small until concurrency limit is reached.", "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 1 }, "id": 31, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tidb_server_get_token_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "99", "refId": "A" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 2 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99% Get Token Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "µs", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 500 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Connection Count alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "TiDB current connection counts", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 8 }, "id": 1, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "total", "fill": 0, "lines": false } ], "spaceLength": 10, "stack": true, "steppedLine": true, "targets": [ { "expr": "tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A", "step": 10 }, { "expr": "sum(tidb_server_connections{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "total", "refId": "B", "step": 10 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 500 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Connection Count", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 1000000000 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" }, { "evaluator": { "params": [ 3000000000 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Heap Memory Usage alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "TiDB heap memory size in use ", "fill": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 8 }, "id": 2, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=~\"tidb.*\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}-{{job}}", "metric": "go_memstats_heap_inuse_bytes", "refId": "A", "step": 10 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1000000000 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Heap Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "TiDB-Server", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 1 }, "id": 34, "panels": [ { "alert": { "conditions": [ { "evaluator": { "params": [ 0.01 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Parse Duration alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "The time cost of parsing SQL to AST", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 2 }, "id": 4, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tidb_session_parse_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, sql_type))", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "{{sql_type}}", "refId": "A", "step": 30 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.01 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99% Parse Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "Parse", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 2 }, "id": 35, "panels": [ { "alert": { "conditions": [ { "evaluator": { "params": [ 0.03 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Compile Duration alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "The time cost of building the query plan", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 3 }, "id": 5, "interval": "", "legend": { "alignAsTable": true, "avg": true, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tidb_session_compile_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, sql_type))", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "{{sql_type}}", "refId": "A", "step": 30 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.03 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99% Compile Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "Compile", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 3 }, "id": 36, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Bucketed histogram of transaction execution durations, including retry", "fill": 1, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 4 }, "id": 30, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99-{{sql_type}}", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95-{{sql_type}}", "refId": "B" }, { "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80-{{sql_type}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 500 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Transaction Statement Num alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "TiDB statements numbers within one transaction. Internal means TiDB inner transaction", "fill": 1, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 4 }, "id": 6, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tidb_session_transaction_statement_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (le, sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99-{{sql_type}}", "refId": "A" }, { "expr": "histogram_quantile(0.80, sum(rate(tidb_session_transaction_statement_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (le, sql_type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "80-{{sql_type}}", "refId": "B" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 500 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Transaction Statement Num", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 3 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Transaction Retry Num alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "TiDB transaction retry histogram bucket statistics", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 4 }, "id": 7, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(1.0, sum(rate(tidb_session_retry_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "100", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.99, sum(rate(tidb_session_retry_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { "expr": "histogram_quantile(0.90, sum(rate(tidb_session_retry_num_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", "refId": "C" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 3 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Transaction Retry Num", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "Transaction", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 4 }, "id": 37, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "kv command durations statistics by command type", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 5 }, "id": 10, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"commit\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KV Cmd Duration 999", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "kv command durations statistics by command type", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 5 }, "id": 11, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_txn_cmd_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"commit\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KV Cmd Duration 99", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 500 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Lock Resolve OPS alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "lock resolve times", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 12 }, "id": 8, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tidb_tikvclient_lock_resolver_actions_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "tidb_tikvclient_lock_resolver_actions_total{}", "refId": "A", "step": 40 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 500 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Lock Resolve OPS", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "kv command durations statistics by command type", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 12 }, "id": 69, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tidb_tikvclient_backoff_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 40 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99% KV Backoff Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 500 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "KV Backoff OPS alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "kv storage backoff times", "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 19 }, "id": 9, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "total": true, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tidb_tikvclient_backoff_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 500 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KV Backoff OPS", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "KV", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, "id": 38, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of a client calling GetTSAsync until received the TS result.", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 6 }, "id": 12, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.99, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { "expr": "histogram_quantile(0.90, sum(rate(pd_client_cmd_handle_cmds_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "PD TSO Wait Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 0.03 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "PD TSO RPC Duration alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of a client sending TSO request until received the response.", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 6 }, "id": 13, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "999", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.99, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "B" }, { "expr": "histogram_quantile(0.90, sum(rate(pd_client_request_handle_requests_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"tso\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", "refId": "C" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.03 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "PD TSO RPC Duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "PD Client", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 6 }, "id": 39, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The execution time of gRPC message", "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "id": 29, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": false, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=~\"kv_prewrite|kv_commit\"}[5m])) by (le, type))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99% gRPC message duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 3.6 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "gRPC poll CPU alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The CPU utilization of gRPC", "fill": 1, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "id": 14, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "topk(5, sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"grpc.*\"}[1m])) by (instance, name))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ instance }} - {{ name }}", "refId": "A", "step": 4 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 3.6 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "gRPC poll CPU", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "gRPC", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, "id": 40, "panels": [ { "alert": { "conditions": [ { "evaluator": { "params": [ 0.02 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Scheduler latch wait duration alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "the time which is caused by latch wait ", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 8 }, "id": 15, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "scopedVars": { "command": { "selected": false, "text": "scan_lock", "value": "scan_lock" } }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99% - {{ type }}", "metric": "", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95% - {{ type }}", "metric": "", "refId": "B", "step": 10 }, { "expr": "rate(tikv_scheduler_latch_wait_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m]) / rate(tikv_scheduler_latch_wait_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg - {{ type }}", "metric": "", "refId": "C", "step": 10 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.02, "yaxis": "left" } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Scheduler latch wait duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 3.6 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Scheduler worker CPU alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The CPU utilization of scheduler worker", "editable": true, "error": false, "fill": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 8 }, "id": 16, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"sched_.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 4 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 3.6 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Scheduler worker CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time consumed when executing commit command", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, "id": 67, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "scopedVars": { "command": { "selected": false, "text": "commit", "value": "commit" } }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ type }}", "metric": "", "refId": "A", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99% scheduler command duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "Scheduler", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, "id": 43, "panels": [ { "alert": { "conditions": [ { "evaluator": { "params": [ 0.05 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Propose wait duration alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each proposal", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 9 }, "id": 19, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ instance }}", "metric": "tikv_raftstore_request_wait_time_duration_secs_bucket{}", "refId": "A", "step": 4 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.05 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99% propose wait duration by instance", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 1.7 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "datasourceId": 1, "model": { "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"raftstore_.*\"}[1m])) by (instance)", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 20 }, "params": [ "A", "1m", "now" ] }, "reducer": { "params": [], "type": "max" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "message": "TiKV raftstore thread CPU usage is high", "name": "TiKV raft store CPU alert", "noDataState": "ok", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The CPU utilization of raftstore thread", "editable": true, "error": false, "fill": 0, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 9 }, "id": 21, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"raftstore_.*\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 4 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.7 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Raft store CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time consumed by processing asynchronous write requests", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "id": 65, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"write\"}[5m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_storage_engine_async_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"write\"}[5m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", "refId": "B", "step": 4 }, { "expr": "sum(rate(tikv_storage_engine_async_request_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"write\"}[5m])) / sum(rate(tikv_storage_engine_async_request_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", type=\"write\"}[5m]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", "refId": "C", "step": 4 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Storage async write duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "raftstore", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, "id": 44, "panels": [ { "alert": { "conditions": [ { "evaluator": { "params": [ 0.05 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Append log duration alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft appends log on each TiKV instance", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 10 }, "id": 32, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_append_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ instance }}", "metric": "", "refId": "A", "step": 4 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.05 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99% append log duration by instance", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time consumed when executing write operation", "fill": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 10 }, "id": 63, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "scopedVars": { "db": { "selected": false, "text": "kv", "value": "kv" } }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"raft\", type=\"write_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", "refId": "A", "step": 10 }, { "expr": "max(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"raft\", type=\"write_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", "refId": "B", "step": 10 }, { "expr": "max(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"raft\", type=\"write_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", "refId": "C", "step": 10 }, { "expr": "max(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"raft\", type=\"write_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Write duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "µs", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "RocksDB-Raft", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, "id": 45, "panels": [ { "alert": { "conditions": [ { "evaluator": { "params": [ 0.1 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "for": "5m", "frequency": "1m", "handler": 1, "name": "Apply wait duration alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when apply log requests are wait for being handled", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 11 }, "id": 49, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])) by (le, instance))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ instance }}", "metric": "tikv_raftstore_request_wait_time_duration_secs_bucket{}", "refId": "A", "step": 4 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.1 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "99% apply wait duration by instance", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 0.05 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "frequency": "60s", "handler": 1, "name": "Apply log duration alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft applies log", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 11 }, "id": 22, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": " 99%", "metric": "", "refId": "A", "step": 4 }, { "expr": "histogram_quantile(0.95, sum(rate(tikv_raftstore_apply_log_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", "refId": "B", "step": 4 }, { "expr": "sum(rate(tikv_raftstore_apply_log_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) / sum(rate(tikv_raftstore_apply_log_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", "refId": "C", "step": 4 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.05 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Apply log duration", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time consumed when executing write operation", "fill": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 18 }, "id": 51, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "scopedVars": { "db": { "selected": false, "text": "kv", "value": "kv" } }, "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=\"write_max\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "max", "refId": "A", "step": 10 }, { "expr": "max(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=\"write_percentile99\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "99%", "refId": "B", "step": 10 }, { "expr": "max(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=\"write_percentile95\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "95%", "refId": "C", "step": 10 }, { "expr": "max(tikv_engine_write_micro_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=\"write_average\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "avg", "refId": "D", "step": 10 } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Write duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "µs", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "alert": { "conditions": [ { "evaluator": { "params": [ 1.8 ], "type": "gt" }, "operator": { "type": "and" }, "query": { "params": [ "A", "5m", "now" ] }, "reducer": { "params": [], "type": "avg" }, "type": "query" } ], "executionErrorState": "alerting", "for": "5m", "frequency": "1m", "handler": 1, "name": "Async apply CPU alert", "noDataState": "no_data", "notifications": [] }, "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The CPU utilization of async apply", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 18 }, "id": 47, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"apply_[0-9]+\"}[1m])) by (instance)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "tikv_thread_cpu_seconds_total", "refId": "A", "step": 4 } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.8 } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Async apply CPU", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "RocksDB-KV", "type": "row" }, { "collapsed": true, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, "id": 53, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, "description": "Shows average latency for Reads and Writes IO Devices. Higher than typical latency for highly loaded storage indicates saturation (overload) and is frequent cause of performance problems. Higher than normal latency also can indicate internal storage problems.", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 12 }, "id": 55, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 1, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "irate(node_disk_write_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m]) / irate(node_disk_writes_completed_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Write: {{ instance }} - {{ device }}", "metric": "", "refId": "A", "step": 300, "target": "" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk Latency", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "s", "label": "", "logBase": 2, "max": null, "min": null, "show": true }, { "format": "s", "label": "", "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, "description": "Shows amount of physical IOs (reads and writes) different devices are serving. Spikes in number of IOs served often corresponds to performance problems due to IO subsystem overload.", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 19 }, "id": 57, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 1, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "irate(node_disk_writes_completed_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Write: {{ instance }} - {{ device }}", "metric": "", "refId": "A", "step": 300, "target": "" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk Operations", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "iops", "label": "", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, "description": "Shows volume of reads and writes the storage is handling. This can be better measure of IO capacity usage for network attached and SSD storage as it is often bandwidth limited. Amount of data being written to the disk can be used to estimate Flash storage life time.", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 26 }, "id": 59, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null as zero", "percentage": false, "pointradius": 1, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "irate(node_disk_written_bytes_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Write: {{ instance }} - {{ device }}", "metric": "", "refId": "A", "step": 300, "target": "" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk Bandwidth", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": "", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 2, "description": "Shows how much disk was loaded for reads or writes as average number of outstanding requests at different period of time. High disk load is a good measure of actual storage utilization. Different storage types handle load differently - some will show latency increases on low loads others can handle higher load with no problems.", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 33 }, "id": 61, "legend": { "alignAsTable": true, "avg": true, "current": false, "hideEmpty": false, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "nullPointMode": "null", "percentage": false, "pointradius": 1, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "calculatedInterval": "2m", "datasourceErrors": {}, "errors": {}, "expr": "irate(node_disk_write_time_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[5m])", "format": "time_series", "interval": "", "intervalFactor": 1, "legendFormat": "Write: {{instance}} - {{ device }}", "metric": "", "refId": "A", "step": 300, "target": "" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Disk Load", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": "", "logBase": 1, "max": null, "min": 0, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Disk", "type": "row" } ], "refresh": "5s", "schemaVersion": 18, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": { }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "K8s-cluster", "multi": false, "name": "k8s_cluster", "options": [ ], "query": "label_values(tikv_engine_block_cache_size_bytes, k8s_cluster)", "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": { }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", "options": [ ], "query": "label_values(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "", "title": "[SharedScope]Test-Cluster-Performance-Write", "uid": "Fcw5wqcWk", "version": 1 }