{ "__inputs": [ { "description": "", "label": "test-cluster", "name": "DS_TEST-CLUSTER", "pluginId": "prometheus", "pluginName": "Prometheus", "type": "datasource" } ], "annotations": { "list": [] }, "description": "", "editable": true, "gnetId": null, "graphTooltip": 1, "hideControls": false, "id": null, "links": [], "panels": [ { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 1, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Write Pipeline Composition", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 2, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Write Raft Log .99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Propose Wait .99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Apply Wait .99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Replicate Raft Log .99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Apply Duration .99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Pipeline Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Read Duration Composition", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 3, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Get Snapshot .99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Cop Wait .99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.95,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Cop Handle .99", "metric": "", "query": "histogram_quantile(0.95,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cop Read Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Duration", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 4, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage size per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 5, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type = \"used\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type = \"used\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The available capacity size of each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 6, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"available\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"available\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Available size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The capacity size per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 7, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Capacity size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU usage of each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 8, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memory usage per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 9, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The I/O utilization per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 10, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{device}}", "metric": "", "query": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IO utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total bytes of read and write in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 11, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-write", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-read", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "MBps", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of leaders on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 12, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of the gRPC message failures", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 13, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-grpc-msg-fail", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"noop\"}\n [$__rate_interval]\n)) by (instance) < 1", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-pd-heartbeat", "metric": "", "query": "sum(delta(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"noop\"}\n [$__rate_interval]\n)) by (instance) < 1", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Errps", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of leaders on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 14, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Leader", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Regions and Buckets on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 15, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"buckets\"}\n \n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-buckets", "metric": "", "query": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"buckets\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "TiKV uptime since the last restart", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 16, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(time() - ((\n process_start_time_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(time() - ((\n process_start_time_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Uptime", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Cluster", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 17, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.0, "yaxis": "left" } ] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 18, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.0, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Critical error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "\nIndicates occurrences of events that make the TiKV instance unavailable\ntemporarily, such as Write Stall, Channel Full, Scheduler Busy, and Coprocessor\nFull", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 19, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "scheduler-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "channelfull-{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"full\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "coprocessor-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"full\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write_stall_percentile99\",db=~\"$db\"}\n \n)) by (instance, db) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "stall-{{instance}}-{{db}}", "metric": "", "query": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write_stall_percentile99\",db=~\"$db\"}\n \n)) by (instance, db) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "store-write-channelfull-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Server is busy", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of reporting failure messages", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 20, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_report_failure_msg_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance, store_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}-to-{{store_id}}", "metric": "", "query": "sum(rate(\n tikv_server_report_failure_msg_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance, store_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Server report failures", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of different raftstore errors on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 21, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status!~\"success|all\"}\n [$__rate_interval]\n)) by (instance, status) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{status}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status!~\"success|all\"}\n [$__rate_interval]\n)) by (instance, status) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raftstore error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of scheduler errors per type on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 22, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"snapshot_err|prepare_write_err\"}\n [$__rate_interval]\n)) by (instance, stage) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{stage}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"snapshot_err|prepare_write_err\"}\n [$__rate_interval]\n)) by (instance, stage) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of different coprocessor errors on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 23, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of gRPC message errors per type on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 24, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC message error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of dropped leaders per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 25, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Leader drop", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of missing leaders per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 26, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_leader_missing\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_leader_missing\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Leader missing", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "RocksDB damaged SST files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 27, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_rocksdb_damaged_files\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-existed", "metric": "", "query": "((\n tikv_rocksdb_damaged_files\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_rocksdb_damaged_files_deleted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-deleted", "metric": "", "query": "((\n tikv_rocksdb_damaged_files_deleted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Damaged files", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of Log Replication Reject caused by follower memory insufficient", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 28, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_append_rejects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_append_rejects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Log Replication Rejected", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Errors", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 29, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The size of each column family", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 30, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum((\n tikv_engine_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CF size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of channel full errors on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 31, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Channel full", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of leaders being written on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 32, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Active written leaders", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 33, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Approximate region size", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The approximate Region size", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 34, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_region_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_region_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Approximate region size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 35, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_region_written_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_region_written_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region written bytes", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average rate of writing bytes to Regions per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 36, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_region_written_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_region_written_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region average written bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 37, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_region_written_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_region_written_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region written keys", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average rate of written keys to Regions per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 38, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_region_written_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_region_written_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region average written keys", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of peers in hibernated state", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 39, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_hibernated_peer_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, state) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{state}}", "metric": "", "query": "sum((\n tikv_raftstore_hibernated_peer_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, state) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Hibernate Peers", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 40, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{name}}", "metric": "", "query": "((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n raft_engine_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-raft-engine", "metric": "", "query": "((\n raft_engine_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory trace", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 41, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raft_entries_evict_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raft_entries_evict_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Entry Cache Evicts", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 42, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_address_resolve_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_address_resolve_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Resolve address duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 43, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{priority}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "99% Thread Pool Schedule Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average rate of written keys to Regions per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 44, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{priority}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Average Thread Pool Schedule Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 45, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}}", "metric": "", "query": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "copr-{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk IO time per second", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ns", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 46, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}}", "metric": "", "query": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "copr-{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk IO bytes per second", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ns", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Server", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 47, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different kinds of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 48, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{priority}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC message count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different kinds of gRPC message which is failed", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 49, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC message failed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The 99% percentile of execution time of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 50, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{priority}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% gRPC message duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average execution time of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 51, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, priority) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, priority) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{priority}}", "metric": "", "query": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, priority) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, priority) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Average gRPC message duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The 99% percentile of execution time of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 52, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_grpc_req_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% request", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_grpc_req_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_grpc_resp_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% response", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_grpc_resp_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_grpc_req_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_server_grpc_req_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg request", "metric": "", "query": "(sum(rate(\n tikv_server_grpc_req_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_server_grpc_req_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_grpc_resp_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_server_grpc_resp_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg response", "metric": "", "query": "(sum(rate(\n tikv_server_grpc_resp_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_server_grpc_resp_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_request_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% kv get batch", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_request_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_request_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_server_request_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg kv batch", "metric": "", "query": "(sum(rate(\n tikv_server_request_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_server_request_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC batch size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 53, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_message_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_message_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_raft_message_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_server_raft_message_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_server_raft_message_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_server_raft_message_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "raft message batch size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The QPS of different sources of gRPC request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 54, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_request_source_counter_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (source) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{source}}", "metric": "", "query": "sum(rate(\n tikv_grpc_request_source_counter_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (source) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC request sources QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of different sources of gRPC request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 55, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_request_source_duration_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (source) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{source}}", "metric": "", "query": "sum(rate(\n tikv_grpc_request_source_duration_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (source) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC request sources duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The QPS of different resource groups of gRPC request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 56, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_resource_group_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "sum(rate(\n tikv_grpc_resource_group_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC resource group QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "gRPC", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 57, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of raftstore thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 58, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft store CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of async apply", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 59, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Async apply CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of store writer thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.8, "yaxis": "left" } ] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 60, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.8, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Store writer CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of gRPC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 61, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC poll CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of scheduler worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 3.6, "yaxis": "left" } ] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 62, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 3.6, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Scheduler worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of readpool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 3.6, "yaxis": "left" } ] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 63, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_norm.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-normal", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_norm.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_high.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-high", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_high.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_low.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-low", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_low.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 3.6, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Storage ReadPool CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of the unified read pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 64, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified_read_po.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified_read_po.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Unified read pool CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of RocksDB", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 65, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"rocksdb.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"rocksdb.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "RocksDB CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of coprocessor", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 66, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_normal.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-normal", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_normal.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_high.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-high", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_high.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_low.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-low", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_low.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Coprocessor CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 67, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"gc_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"gc_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 68, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"background.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"background.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Background Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 69, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog_fetch.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog_fetch.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raftlog fetch Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 70, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 71, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(backup-worker|bkwkr|backup_endpoint).*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(backup-worker|bkwkr|backup_endpoint).*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 72, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdcwkr.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-worker", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdcwkr.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-tso", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdc_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-endpoint", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdc_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CDC worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of raftstore thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 73, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso_worker\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso_worker\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TSO Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Thread CPU", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 74, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 75, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_expire_kv_count_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_ttl_expire_kv_count_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL expire count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 76, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_expire_kv_size_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_ttl_expire_kv_size_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL expire size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 77, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_ttl_checker_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_ttl_checker_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL check progress", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 78, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_checker_actions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_ttl_checker_actions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL checker actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing GC tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 79, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_ttl_checker_compact_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_ttl_checker_compact_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL checker compact duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 80, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_ttl_checker_poll_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"tikv_gc_run_interval\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": null, "metric": "", "query": "max((\n tikv_ttl_checker_poll_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"tikv_gc_run_interval\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "TTL checker poll interval", "transformations": [], "transparent": false, "type": "stat" } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "TTL", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 81, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of requests that TiKV sends to PD", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 82, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by requests that TiKV sends to PD", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 83, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD request duration (average)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of PD heartbeat messages", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 84, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_pd_pending_heartbeat_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-pending", "metric": "", "query": "sum((\n tikv_pd_pending_heartbeat_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD heartbeats", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of peers validated by the PD worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 85, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_pd_validate_peer_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_pd_validate_peer_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD validate peers", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of reconnection between TiKV and PD", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 86, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_pd_reconnect_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(delta(\n tikv_pd_reconnect_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD reconnection", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The forward status of PD client", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 87, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_pd_request_forwarded\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{host}}", "metric": "", "query": "((\n tikv_pd_request_forwarded\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD forward status", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of TSO requests waiting in the queue.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 88, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_pd_pending_tso_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_pd_pending_tso_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Pending TSO Requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The slow score of stores", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 89, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_slow_score\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_slow_score\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store Slow Score", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration that recorded by inspecting messages.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 90, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_inspect_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_inspect_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Inspected duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "PD", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 91, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The throughput of disk write per IO type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 92, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write IO bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The throughput of disk read per IO type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 93, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read IO bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The threshold of disk IOs per priority", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 94, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_rate_limiter_max_bytes_per_sec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "avg((\n tikv_rate_limiter_max_bytes_per_sec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IO threshold", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "IO rate limiter request wait duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 95, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Rate Limiter Request Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "IO Breakdown", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 96, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by processing asynchronous write requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 97, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The store time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 98, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The apply time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 99, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_apply_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_apply_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The propose wait time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 100, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store propose wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The batch wait time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 101, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store batch wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The send-to-write-queue time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 102, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store send to write queue duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The send raft message of the proposal duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 103, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store send proposal duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The write kv db end duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 104, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store write kv db end duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The before write time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 105, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store before write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The persist duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 106, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store persist duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The write end duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 107, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store write end duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The commit but not persist duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 108, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store commit but not persist duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The commit and persist duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 109, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store commit and persist duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Waterfall", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 110, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed for peer processes to be ready in Raft", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 111, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Process ready duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed for peer processes to be ready in Raft", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 112, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_raft_process_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_raft_process_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Process ready duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time duration of store write loop when store-io-pool-size is not zero.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 113, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write loop duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time duration of store write loop on each TiKV instance when store-io-pool-size is not zero.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 114, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Store write loop duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft appends log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 115, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Append log duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft appends log on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 116, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_append_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_append_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_append_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_append_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_append_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_append_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Append log duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft commits log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 117, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Commit log duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft commits log on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 118, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Commit log duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft applies log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 119, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply log duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed for Raft to apply logs per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 120, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_apply_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_apply_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Apply log duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed for Raft Client wait connection ready", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 121, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Raft Client Wait Connection Ready Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed for Raft Client wait connection ready per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 122, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{to}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{to}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_raft_client_wait_ready_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to) / sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{to}}", "metric": "", "query": "(sum(rate(\n tikv_server_raft_client_wait_ready_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to) / sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{to}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Raft Client Wait Connection Ready Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The throughput of disk write per IO type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 123, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_io_reschedule_region_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "rechedule-{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_io_reschedule_region_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_io_reschedule_pending_tasks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "pending-task-{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_io_reschedule_pending_tasks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store io task reschedule", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when store write task block on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 124, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Write task block duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft IO", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 125, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The proposal count of a Regions in a tick", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 126, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_proposal_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_proposal_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft proposals per ready", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of proposals per type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 127, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|normal|read_index\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|normal|read_index\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft read/write proposals", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of read proposals which are made by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 128, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|read_index\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|read_index\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft read proposals per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of write proposals which are made by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 129, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"normal\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"normal\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft write proposals per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each proposal", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 130, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Propose wait duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each proposal in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 131, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Propose wait duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each store write task", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 132, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write wait duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each store write task in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 133, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Store write wait duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each apply task", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 134, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply wait duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each apply task in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 135, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Apply wait duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The handle duration of each store write task msg", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 136, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_handle_msg_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_handle_msg_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write handle msg duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The distribution of write trigger size", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 137, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_trigger_wb_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_trigger_wb_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write trigger size", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate at which peers propose logs", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 138, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_propose_log_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_propose_log_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft propose speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate at which peers propose logs", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 139, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "store-{{type}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "apply-{{type}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Perf Context duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Propose", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 140, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different ready type of Raft", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 141, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_ready_handled_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_ready_handled_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Ready handled", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The max time consumed by raftstore events", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 142, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_raftstore_event_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_raftstore_event_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_broadcast_normal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "broadcast_normal", "metric": "", "query": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_broadcast_normal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max duration of raft store events", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Replica read lock checking duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 143, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_replica_read_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_replica_read_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Replica read lock checking duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The length of peer msgs for each round handling", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 144, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_peer_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_peer_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Peer msg length distribution", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Process", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 145, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Raft messages sent by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 146, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Sent messages per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Raft messages flushed by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 147, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_message_flush_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_message_flush_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flush messages per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Raft messages received by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 148, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_message_recv_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_message_recv_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Receive messages per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of different types of Raft messages that are sent", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 149, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Messages", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of vote messages that are sent in Raft", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 150, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"vote\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"vote\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Vote", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of dropped Raft messages per type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 151, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_dropped_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_dropped_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft dropped messages", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Message", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 152, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of admin proposals", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 153, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"conf_change|transfer_leader\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"conf_change|transfer_leader\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Admin proposals", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of the processed apply command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 154, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_admin_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"compact\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_admin_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"compact\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Admin apply", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of raftstore split checks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 155, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_check_split_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"ignore\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_check_split_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"ignore\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Check split", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when running split check in .9999", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 156, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_check_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_check_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% Check split duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 157, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_load_base_split_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(delta(\n tikv_load_base_split_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Load base split event", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 158, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.8,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "80%-{{instance}}", "metric": "", "query": "histogram_quantile(0.8,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_load_base_split_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_load_base_split_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_load_base_split_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_load_base_split_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Load base split duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 159, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_peer_in_flashback_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_peer_in_flashback_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Peer in Flashback State", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Admin", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 160, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 161, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log GC write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 162, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log GC kv sync duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 163, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log GC write operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 164, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_log_gc_seek_operations_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_log_gc_seek_operations_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log GC seek operations ", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 165, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_log_lag_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_log_lag_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log lag", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 166, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_log_gc_skipped\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_log_gc_skipped\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log gc skipped", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 167, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_log_gc_failed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_log_gc_failed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log GC failed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 168, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_entry_fetches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_entry_fetches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log fetch ", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 169, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/pending-task/", "bars": false, "fill": 1, "fillBelowTo": null, "lines": true, "transform": "negative-Y", "yaxis": 2, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog-fetch-worker\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "pending-task", "metric": "", "query": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog-fetch-worker\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log async fetch task duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Log", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 170, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 171, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/.*-total/", "bars": false, "fill": 1, "fillBelowTo": null, "lines": true, "yaxis": 2, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_local_read_reject_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-reject-by-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_local_read_reject_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_local_read_executed_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-total", "metric": "", "query": "sum(rate(\n tikv_raftstore_local_read_executed_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_local_read_executed_stale_read_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-stale-read", "metric": "", "query": "sum(rate(\n tikv_raftstore_local_read_executed_stale_read_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log async fetch task duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Local Reader", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 172, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time used by each level in the unified read pool per second. Level 0 refers to small queries.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 173, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=\"unified-read-pool\"}\n [$__rate_interval]\n)) by (level) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{level}}", "metric": "", "query": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=\"unified-read-pool\"}\n [$__rate_interval]\n)) by (level) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Time used by level", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The chance that level 0 (small) tasks are scheduled in the unified read pool.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 174, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=\"unified-read-pool\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=\"unified-read-pool\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Level 0 chance", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of concurrently running tasks in the unified read pool.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 175, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(avg_over_time(\n tikv_unified_read_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (priority) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{priority}}", "metric": "", "query": "sum(avg_over_time(\n tikv_unified_read_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (priority) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Running tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 176, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Unified Read Pool Wait Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Unified read pool task execution time during one schedule.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 177, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Duration of One Time Slice", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Unified read pool task total execution duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 178, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_exec_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_exec_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Execute Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Task schedule number of times.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 179, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_execute_times_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_execute_times_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Schedule Times", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Unified Read Pool", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 180, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total count of different kinds of commands received", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 181, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_command_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_storage_command_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage command total", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of engine asynchronous request errors", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 182, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status!~\"all|success\"}\n [$__rate_interval]\n)) by (status) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{status}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status!~\"all|success\"}\n [$__rate_interval]\n)) by (status) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async request error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by processing asynchronous write requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 183, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Storage async write duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async write duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 184, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by processing asynchronous snapshot requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 185, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async snapshot duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 186, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async snapshot duration without the involving of raftstore", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 187, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration (pure local read)", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async snapshot duration without the involving of raftstore", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 188, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration (pure local read)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Read index propose wait duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 189, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Read index propose wait duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Read index propose wait duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 190, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read index propose wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Read index confirm duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 191, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Read index confirm duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Read index confirm duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 192, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read index confirm duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "CPU usage measured over a 30 second window", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 193, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_storage_process_stat_cpu_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_storage_process_stat_cpu_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Process Stat Cpu Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 194, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_full_compact_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_storage_full_compact_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Full compaction duration seconds", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 195, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Full compaction pause duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 196, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Full compaction per-increment duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Storage", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 197, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 198, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) != 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "throttle-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) != 0", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 199, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_discard_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) / 10000000", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum((\n tikv_scheduler_discard_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) / 10000000", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler discard ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 200, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Throttle duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 201, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_scheduler_throttle_cf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) != 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{cf}}", "metric": "", "query": "((\n tikv_scheduler_throttle_cf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) != 0", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler throttled CF", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 202, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_throttle_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{cf}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_throttle_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, cf) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flow controller actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 203, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}_l0_flow-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}_flush_flow-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total_l0_flow-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total_flush_flow-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flush/L0 flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 204, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_scheduler_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "l0-{{instance}}", "metric": "", "query": "max((\n tikv_scheduler_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_scheduler_memtable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "memtable-{{instance}}", "metric": "", "query": "max((\n tikv_scheduler_memtable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_scheduler_l0_avg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg_l0-{{instance}}", "metric": "", "query": "max((\n tikv_scheduler_l0_avg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flow controller factors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 205, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n \n)) by (cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}", "metric": "", "query": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n \n)) by (cf) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (cf) / 10000000", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "pending-bytes-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (cf) / 10000000", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction pending bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Throttle time for txn storage commands in 1 minute.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 206, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Txn command throttled duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Throttle time for non-txn related processing like analyze or dag in 1 minute.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 207, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_non_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_non_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Non-txn command throttled duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Flow Control", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 208, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of commands on each stage in commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 209, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "busy-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (stage) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{stage}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (stage) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler stage total", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 210, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_scheduler_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler command duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time which is caused by latch wait in commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 211, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler latch wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of keys read by a commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 212, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_kv_command_key_read_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_scheduler_kv_command_key_read_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler keys read", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of keys written by a commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 213, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_kv_command_key_write_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_scheduler_kv_command_key_write_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler keys written", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of each CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 214, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\"}\n [$__rate_interval]\n)) by (tag) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\"}\n [$__rate_interval]\n)) by (tag) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of lock CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 215, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"lock\"}\n [$__rate_interval]\n)) by (tag) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"lock\"}\n [$__rate_interval]\n)) by (tag) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details [lock]", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of write CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 216, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"write\"}\n [$__rate_interval]\n)) by (tag) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"write\"}\n [$__rate_interval]\n)) by (tag) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details [write]", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of default CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 217, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"default\"}\n [$__rate_interval]\n)) by (tag) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"default\"}\n [$__rate_interval]\n)) by (tag) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details [default]", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on reading when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 218, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_processing_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_scheduler_processing_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler command read duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on checking memory locks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 219, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_check_mem_lock_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_storage_check_mem_lock_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Check memory locks duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null } ], "repeat": "command", "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Scheduler - $command", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 220, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of commands on each stage", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 221, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{stage}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{stage}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler stage total", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total writing bytes of commands on each stage", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 222, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler writing bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different priority commands", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 223, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (priority) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{priority}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (priority) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler priority commands", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of pending commands per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 224, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_contex_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_contex_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler pending commands", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 225, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Txn Scheduler Pool Wait Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Scheduler", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 226, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of GC tasks processed by gc_worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 227, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_tasks_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-{{task}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_tasks_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_gc_skipped_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "skipped-{{task}}", "metric": "", "query": "sum(rate(\n tikv_storage_gc_skipped_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_task_fail_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "failed-{{task}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_task_fail_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_worker_too_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "gcworker-too-busy", "metric": "", "query": "sum(rate(\n tikv_gc_worker_too_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing GC tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 228, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_gcworker_gc_task_duration_vec_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_gcworker_gc_task_duration_vec_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC tasks duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The GC duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 229, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1,(\n sum(rate(\n tidb_tikvclient_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(1,(\n sum(rate(\n tidb_tikvclient_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiDB GC seconds", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of TiDB GC worker actions", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 230, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_tikvclient_gc_worker_actions_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tidb_tikvclient_gc_worker_actions_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiDB GC worker actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Progress of ResolveLocks, the first phase of GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 231, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_tikvclient_range_task_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=~\"resolve-locks.*\"}\n \n)) by (result) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{result}}", "metric": "", "query": "max((\n tidb_tikvclient_range_task_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=~\"resolve-locks.*\"}\n \n)) by (result) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "ResolveLocks Progress", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Progress of TiKV's GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 232, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum((\n tikv_gcworker_autogc_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"}\n \n)) by (instance) / sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n \n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum((\n tikv_gcworker_autogc_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"}\n \n)) by (instance) / sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n \n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiKV Auto GC Progress", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "keys / second", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 233, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_mvcc_gc_delete_versions_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_keys/s", "metric": "", "query": "sum(rate(\n tikv_storage_mvcc_gc_delete_versions_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "SafePoint used for TiKV's Auto GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 234, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_gcworker_autogc_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance) / (2^18)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "max((\n tikv_gcworker_autogc_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance) / (2^18)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiKV Auto GC SafePoint", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The lifetime of TiDB GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 235, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=\"tikv_gc_life_time\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": null, "metric": "", "query": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=\"tikv_gc_life_time\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "GC lifetime", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The interval of TiDB GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 236, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=\"tikv_gc_run_interval\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": null, "metric": "", "query": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=\"tikv_gc_run_interval\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "GC interval", "transformations": [], "transparent": false, "type": "stat" }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Keys handled in GC compaction filter", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 237, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_filtered", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_skip\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_skipped", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_skip\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_mvcc_rollback\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-rollback/mvcc-lock", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_mvcc_rollback\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_orphan_versions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_orphan-versions", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_orphan_versions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_perform\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_performed-times", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_perform\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_failure-{{type}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_met\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-deletion-met", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_met\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_handled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-deletion-handled", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_handled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_wasted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-deletion-wasted", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_wasted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC in Compaction Filter", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "GC scan write details", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 238, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"}\n [$__rate_interval]\n)) by (key_mode, tag) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}-{{tag}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"}\n [$__rate_interval]\n)) by (key_mode, tag) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC scan write details", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "GC scan default details", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 239, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"}\n [$__rate_interval]\n)) by (key_mode, tag) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}-{{tag}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"}\n [$__rate_interval]\n)) by (key_mode, tag) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC scan default details", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "GC", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 240, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate of Raft snapshot messages sent", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 241, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [1m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [1m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Rate snapshot message", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of snapshots in different states", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 242, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_snapshot_traffic_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum((\n tikv_raftstore_snapshot_traffic_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot state count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time snapshot generation tasks waited to be scheduled. ", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 243, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Snapshot generation wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when handling snapshots", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 244, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_send_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "send", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_send_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "apply", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"generate\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "generate", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"generate\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Handle snapshot duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The snapshot size (P99.99).9999", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 245, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "size", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% Snapshot size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of KV within a snapshot in .9999", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 246, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_kv_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_kv_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% Snapshot KV count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Action stats for snapshot generating and applying", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 247, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_snapshot_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, status) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{status}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_snapshot_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, status) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_clean_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, status) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "clean-region-by-{{type}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_clean_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, status) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot Actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The speed of sending or receiving snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 248, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_snapshot_limit_transport_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_snapshot_limit_transport_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_snapshot_limit_generate_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-generate", "metric": "", "query": "sum(rate(\n tikv_snapshot_limit_generate_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot transport speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Snapshot", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 249, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of tasks handled by worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 250, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Worker handled tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current pending and running tasks of worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 251, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Worker pending tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of tasks handled by future_pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 252, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FuturePool handled tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current pending and running tasks of future_pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 253, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FuturePool pending tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Task", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 254, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle coprocessor read requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 255, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Request duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle coprocessor read requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 256, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 257, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 258, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{reason}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Request Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 259, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "KV Cursor Operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 260, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "KV Cursor Operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 261, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "delete_skipped-{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total RocksDB Perf Statistics", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 262, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Response Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor Overview", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 263, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when handling coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 264, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Handle duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle coprocessor requests per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 265, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}}-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Handle duration by store", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when coprocessor requests are wait for being handled", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 266, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when coprocessor requests are wait for being handled in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 267, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}}-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Wait duration by store", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 268, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (vec_type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{vec_type}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (vec_type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total DAG Requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of DAG executors", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 269, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total DAG Executors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 270, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"select\"}\n [$__rate_interval]\n)) by (tag) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"select\"}\n [$__rate_interval]\n)) by (tag) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details (Table Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 271, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"index\"}\n [$__rate_interval]\n)) by (tag) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"index\"}\n [$__rate_interval]\n)) by (tag) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details (Index Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 272, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"select\"}\n [$__rate_interval]\n)) by (cf, tag) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{tag}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"select\"}\n [$__rate_interval]\n)) by (cf, tag) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details by CF (Table Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 273, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"index\"}\n [$__rate_interval]\n)) by (cf, tag) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{tag}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"index\"}\n [$__rate_interval]\n)) by (cf, tag) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details by CF (Index Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on checking memory locks for coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 274, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Memory lock checking duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on checking memory locks for coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 275, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory lock checking duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor Detail", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 276, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 277, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, state) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{state}}", "metric": "", "query": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, state) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-total", "metric": "", "query": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads state", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 278, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n sum(rate(\n tikv_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, io) > 1024\n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "topk(20,(\n sum(rate(\n tikv_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, io) > 1024\n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads IO", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 279, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n max(rate(\n tikv_thread_voluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "topk(20,(\n max(rate(\n tikv_thread_voluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Thread Voluntary Context Switches", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 280, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n max(rate(\n tikv_thread_nonvoluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "topk(20,(\n max(rate(\n tikv_thread_nonvoluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Thread Nonvoluntary Context Switches", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Threads", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 281, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of get operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 282, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "memtable", "metric": "", "query": "sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"block_cache_data_hit|block_cache_filter_hit\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "block_cache", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"block_cache_data_hit|block_cache_filter_hit\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l0\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "l0", "metric": "", "query": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l0\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l1\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "l1", "metric": "", "query": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l1\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l2_and_up\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "l2_and_up", "metric": "", "query": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l2_and_up\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Get operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing get operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 283, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Get duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of seek operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 284, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seek", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek_found\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seek_found", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek_found\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "next", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next_found\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "next_found", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next_found\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "prev", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev_found\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "prev_found", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev_found\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Seek operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing seek operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 285, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Seek duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of write operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 286, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"write_done_by_self|write_done_by_other\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "done", "metric": "", "query": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"write_done_by_self|write_done_by_other\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_timeout\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "timeout", "metric": "", "query": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_timeout\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_with_wal\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "with_wal", "metric": "", "query": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_with_wal\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 287, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of WAL sync operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 288, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_wal_file_synced\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "sync", "metric": "", "query": "sum(rate(\n tikv_engine_wal_file_synced\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "WAL sync operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing write wal operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 289, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write WAL duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of compaction and flush operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 290, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_event_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_engine_event_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing WAL sync operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 291, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "WAL sync duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Compaction guard actions", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 292, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_compaction_guard_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=~\"default|write\"}\n [$__rate_interval]\n)) by (cf, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{ type}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_compaction_guard_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=~\"default|write\"}\n [$__rate_interval]\n)) by (cf, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction guard actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing the compaction and flush operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 293, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when reading SST files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 294, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "SST read duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 295, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_compaction_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (cf, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_engine_compaction_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (cf, reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The block cache size. Broken down by column family if shared block cache is disabled.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 296, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n avg((\n tikv_engine_block_cache_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, instance) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{cf}}", "metric": "", "query": "topk(20,(\n avg((\n tikv_engine_block_cache_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, instance) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block cache size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The hit rate of memtable", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 297, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_miss\"}\n [$__rate_interval]\n)) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "hit", "metric": "", "query": "(sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_miss\"}\n [$__rate_interval]\n)) ))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memtable hit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The flow of different kinds of block cache operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 298, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_read\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total_read", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_read\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_write\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total_written", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_write\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_bytes_insert\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "data_insert", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_bytes_insert\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_insert\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "filter_insert", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_insert\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_evict\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "filter_evict", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_evict\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_insert\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "index_insert", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_insert\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_evict\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "index_evict", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_evict\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block cache flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The hit rate of block cache", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 299, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_miss\"}\n [$__rate_interval]\n)) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "all", "metric": "", "query": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_miss\"}\n [$__rate_interval]\n)) ))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_miss\"}\n [$__rate_interval]\n)) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "data", "metric": "", "query": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_miss\"}\n [$__rate_interval]\n)) ))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_miss\"}\n [$__rate_interval]\n)) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "filter", "metric": "", "query": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_miss\"}\n [$__rate_interval]\n)) ))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_miss\"}\n [$__rate_interval]\n)) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "index", "metric": "", "query": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_miss\"}\n [$__rate_interval]\n)) ))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bloom_prefix_useful\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bloom_prefix_checked\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "bloom prefix", "metric": "", "query": "(sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bloom_prefix_useful\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bloom_prefix_checked\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block cache hit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The flow of different kinds of operations on keys", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 300, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_read\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_read\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_written\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "written", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_written\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_compaction_num_corrupt_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "corrupt", "metric": "", "query": "sum(rate(\n tikv_engine_compaction_num_corrupt_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keys flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different kinds of block cache operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 301, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total_add", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_add\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "data_add", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_add\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_add\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "filter_add", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_add\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_add\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "index_add", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_add\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add_failures\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "add_failures", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add_failures\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block cache operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The flow rate of read operations per type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 302, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "get", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"iter_bytes_read\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "scan", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"iter_bytes_read\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of keys in each column family", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 303, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_estimate_num_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}", "metric": "", "query": "sum((\n tikv_engine_estimate_num_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total keys", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The flow of different kinds of write operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 304, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "wal", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The bytes per read", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 305, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Bytes / Read", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The flow rate of compaction operations per type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 306, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_compaction_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read", "metric": "", "query": "sum(rate(\n tikv_engine_compaction_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_compaction_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "written", "metric": "", "query": "sum(rate(\n tikv_engine_compaction_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"flush_write_bytes\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "flushed", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"flush_write_bytes\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The bytes per write", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 307, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Bytes / Write", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The read amplification per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 308, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_read_amp_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_total_read_bytes\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_engine_read_amp_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_estimate_useful_bytes\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_engine_read_amp_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_total_read_bytes\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_engine_read_amp_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_estimate_useful_bytes\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read amplification", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The pending bytes to be compacted", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 309, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}", "metric": "", "query": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction pending bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of snapshot of each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 310, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_engine_num_snapshots\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_engine_num_snapshots\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Number of snapshots", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The compression ratio of each level", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 311, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_compression_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, level) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-L{{level}}", "metric": "", "query": "avg((\n tikv_engine_compression_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, level) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compression ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of SST files for different column families in each level", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 105 }, "height": null, "hideTimeOverride": false, "id": 312, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_num_files_at_level\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, level) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-L{{level}}", "metric": "", "query": "avg((\n tikv_engine_num_files_at_level\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, level) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Number files at each level", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time that the oldest unreleased snapshot survivals", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 105 }, "height": null, "hideTimeOverride": false, "id": 313, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_engine_oldest_snapshot_duration\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_engine_oldest_snapshot_duration\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Oldest snapshots duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Stall conditions changed of each column family", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 112 }, "height": null, "hideTimeOverride": false, "id": 314, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_engine_stall_conditions_changed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{cf}}-{{type}}", "metric": "", "query": "((\n tikv_engine_stall_conditions_changed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Stall conditions changed of each CF", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when ingesting SST files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 112 }, "height": null, "hideTimeOverride": false, "id": 315, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_ingest_sst_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_ingest_sst_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_snapshot_ingest_sst_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_snapshot_ingest_sst_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_snapshot_ingest_sst_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_snapshot_ingest_sst_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_snapshot_ingest_sst_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_snapshot_ingest_sst_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_snapshot_ingest_sst_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_snapshot_ingest_sst_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Ingest SST duration seconds", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 119 }, "height": null, "hideTimeOverride": false, "id": 316, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_write_stall_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_engine_write_stall_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Stall Reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time which is caused by write stall", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 119 }, "height": null, "hideTimeOverride": false, "id": 317, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write stall duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The level that the external file ingests into", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 126 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 318, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_ingestion_picked_level_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_engine_ingestion_picked_level_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Ingestion picked level", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memtable size of each column family", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 126 }, "height": null, "hideTimeOverride": false, "id": 319, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"mem-tables-all\"}\n \n)) by (cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}", "metric": "", "query": "avg((\n tikv_engine_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"mem-tables-all\"}\n \n)) by (cf) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memtable size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": "db", "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "RocksDB - $db", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 320, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of operations per second", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 321, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_write_apply_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write", "metric": "", "query": "sum(rate(\n raft_engine_write_apply_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_read_entry_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read_entry", "metric": "", "query": "sum(rate(\n raft_engine_read_entry_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_read_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read_message", "metric": "", "query": "sum(rate(\n raft_engine_read_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Operation", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time used in write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 322, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n raft_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n raft_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The I/O flow rate", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 323, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write", "metric": "", "query": "sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_background_rewrite_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "rewrite-{{type}}", "metric": "", "query": "sum(rate(\n raft_engine_background_rewrite_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "99% duration breakdown of write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 324, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_preprocess_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "wait", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_preprocess_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "wal", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "apply", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Duration Breakdown (99%)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The bytes per write", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 325, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Bytes / Written", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "999% duration breakdown of WAL write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 326, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_sync_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "sync", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_sync_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_allocate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "allocate", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_allocate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_rotate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "rotate", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_rotate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "WAL Duration Breakdown (999%)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average number of files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 327, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_log_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "avg((\n raft_engine_log_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_swap_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "swap", "metric": "", "query": "avg((\n raft_engine_swap_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_recycled_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-recycle", "metric": "", "query": "avg((\n raft_engine_recycled_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "File Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The 99% duration of operations other than write", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 328, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_entry_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read_entry", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_entry_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read_message", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_purge_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "purge", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_purge_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Other Durations (99%)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average number of log entries", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 329, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_log_entry_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "avg((\n raft_engine_log_entry_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Entry Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Engine", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 330, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 331, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_titandb_num_live_blob_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "live blob file num", "metric": "", "query": "sum((\n tikv_engine_titandb_num_live_blob_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_titandb_num_obsolete_blob_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "obsolete blob file num", "metric": "", "query": "sum((\n tikv_engine_titandb_num_obsolete_blob_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 332, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_titandb_live_blob_file_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "live blob file size", "metric": "", "query": "sum((\n tikv_engine_titandb_live_blob_file_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_titandb_obsolete_blob_file_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "obsolete blob file size", "metric": "", "query": "sum((\n tikv_engine_titandb_obsolete_blob_file_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 333, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_titandb_live_blob_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "live blob size", "metric": "", "query": "sum((\n tikv_engine_titandb_live_blob_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Live blob size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The hit rate of block cache", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 334, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_miss\"}\n [$__rate_interval]\n)) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "all", "metric": "", "query": "(sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"}\n [$__rate_interval]\n)) / (sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"}\n [$__rate_interval]\n)) + sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_miss\"}\n [$__rate_interval]\n)) ))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob cache hit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 335, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Iter touched blob file count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The blob cache size.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 336, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n avg((\n tikv_engine_blob_cache_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (cf, instance) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{cf}}", "metric": "", "query": "topk(20,(\n avg((\n tikv_engine_blob_cache_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (cf, instance) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob cache size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 337, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob key size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 338, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob value size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 339, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_get\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "get", "metric": "", "query": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_get\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob get operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 340, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}}", "metric": "", "query": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%-{{type}}", "metric": "", "query": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}}", "metric": "", "query": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max-{{type}}", "metric": "", "query": "max((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob get duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 341, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_titandb_blob_file_discardable_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (ratio) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ratio}}", "metric": "", "query": "sum((\n tikv_engine_titandb_blob_file_discardable_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (ratio) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file discardable ratio distribution", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 342, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_seek\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seek", "metric": "", "query": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_seek\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_prev\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "prev", "metric": "", "query": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_prev\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_next\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "next", "metric": "", "query": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_next\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob iter operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 343, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob seek duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 344, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob next duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 345, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}}", "metric": "", "query": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%-{{type}}", "metric": "", "query": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}}", "metric": "", "query": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max-{{type}}", "metric": "", "query": "max((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob prev duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 346, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob keys flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 347, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_average\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_average\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile99\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile99\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile95\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile95\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_max\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_max\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file read duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 348, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob bytes flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 349, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_average\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_average\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile99\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile99\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile95\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile95\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_max\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_max\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 350, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_file_synced\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "sync", "metric": "", "query": "sum(rate(\n tikv_engine_blob_file_synced\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file sync operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 351, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_gc_action_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_gc_action_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC action", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 352, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_average\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_average\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile95\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile95\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile99\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile99\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_max\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_max\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file sync duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 353, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_average\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_average\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile95\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile95\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile99\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile99\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_max\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_max\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 354, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_gc_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_gc_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC keys flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 355, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC input file size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 356, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_gc_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_gc_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC bytes flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 357, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_average\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_average\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile95\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%", "metric": "", "query": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile95\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile99\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile99\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_max\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max", "metric": "", "query": "max((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_max\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC output file size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 358, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_gc_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_gc_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC file count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": "titan_db", "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Titan - $titan_db", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 359, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 360, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"waiter_manager.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{name}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"waiter_manager.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"deadlock_detect.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{name}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"deadlock_detect.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock Manager Thread CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 361, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_task_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_lock_manager_task_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock Manager Handled tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 362, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Waiter lifetime duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 363, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(max_over_time(\n tikv_lock_manager_wait_table_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(max_over_time(\n tikv_lock_manager_wait_table_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(max_over_time(\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(max_over_time(\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock Waiting Queue", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 364, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_lock_manager_detect_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_lock_manager_detect_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Deadlock detect duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 365, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_lock_manager_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Detect error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 366, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(max_over_time(\n tikv_lock_manager_detector_leader_heartbeat\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(max_over_time(\n tikv_lock_manager_detector_leader_heartbeat\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Deadlock detector leader", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 367, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_pessimistic_lock_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_pessimistic_lock_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total pessimistic locks memory size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 368, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_pessimistic_locking\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (result) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{result}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_pessimistic_locking\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (result) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "In-memory pessimistic locking result", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of active keys and waiters.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 369, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum((\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Pessimistic lock activities", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The length includes the entering transaction itself", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 370, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_wait_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_lock_wait_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Lengths of lock wait queues when transaction enqueues", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration scan in-memory pessimistic locks with read lock", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 371, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{type}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}}", "metric": "", "query": "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{type}}", "metric": "", "query": "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "In-memory scan lock read duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Pessimistic Locking", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 372, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 373, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 374, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"queue|exec_download\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "(DL){{type}}-99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"queue|exec_download\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "(AP){{type}}-99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "P99 RPC Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 375, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (instance, request) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{request}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (instance, request) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-{{request}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Ops", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 376, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_apply_cache_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_apply_cache_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cache Events", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 377, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Overall RPC Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 378, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"exec_download\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"exec_download\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Read File into Memory Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 379, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"queuing\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"queuing\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Queuing Time", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 380, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_apply_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_apply_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply Request Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 381, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_download_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_download_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Downloaded File Size", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 382, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_apply_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_apply_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply Batch Size", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 383, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"get_permit\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"get_permit\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Blocked by Concurrency Time", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 384, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply Request Speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 385, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_import_apply_cached_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_import_apply_cached_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cached File in Memory", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 386, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Engine Requests Unfinished", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 387, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply Time", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 388, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Store Memory Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Point In Time Restore", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 389, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of resolved ts worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 390, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"resolved_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"resolved_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Resolved TS Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of advance ts worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 391, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"advance_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"advance_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Advance ts Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of scan lock worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 392, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"inc_scan.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"inc_scan.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scan lock Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The gap between resolved ts (the maximum candidate of safe-ts) and current time.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 393, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max gap of resolved-ts", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The gap between now() and the minimal (non-zero) safe ts for followers", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 394, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_follower_safe_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_follower_safe_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max gap of follower safe-ts", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The region that has minimal resolved ts", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 395, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Min Resolved TS Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The region id of the follower that has minimal safe ts", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 396, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_follower_safe_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_follower_safe_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Min Safe TS Follower Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when handle a check leader request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 397, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_resolved_ts_check_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_resolved_ts_check_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Check leader duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The gap between resolved ts of leaders and current time", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 398, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max gap of resolved-ts in region leaders", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Bucketed histogram of region count in a check leader request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 399, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% CheckLeader request region count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The backoff duration before starting initial scan", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 400, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Initial scan backoff duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total bytes in memory of resolved-ts observe regions's lock heap", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 401, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_resolved_ts_lock_heap_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "avg((\n tikv_resolved_ts_lock_heap_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock heap size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The region that its leader has minimal resolved ts.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 402, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Min Leader Resolved TS Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The status of resolved-ts observe regions", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 403, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_region_resolve_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum((\n tikv_resolved_ts_region_resolve_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Observe region status", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of fail to advance resolved-ts", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 404, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_resolved_ts_fail_advance_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{reason}}", "metric": "", "query": "sum(delta(\n tikv_resolved_ts_fail_advance_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_check_stale_peer\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-stale-peer", "metric": "", "query": "sum(delta(\n tikv_raftstore_check_stale_peer\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Fail advance ts count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Bucketed histogram of the check leader request size", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 405, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-check-num", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% CheckLeader request size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total bytes of pending commands in the channel", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 406, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_resolved_ts_channel_penging_cmd_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "avg((\n tikv_resolved_ts_channel_penging_cmd_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Pending command size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Resolved TS", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 407, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 408, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_allocator_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n tikv_allocator_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Allocator Stats", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 409, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name) - sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{thread_name}}", "metric": "", "query": "(sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name) - sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Send Allocated(+) / Release Received(-) Bytes Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 410, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{thread_name}}", "metric": "", "query": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Newly Allocated Bytes by Thread", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 411, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{thread_name}}", "metric": "", "query": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Recently Released Bytes by Thread", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Memory", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 412, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 413, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"b.*k.*w.*k.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "backup-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"b.*k.*w.*k.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_io\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "backup-io-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_io\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_backup_softlimit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "backup-auto-throttle-{{instance}}", "metric": "", "query": "((\n tikv_backup_softlimit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup CPU Utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 414, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_backup_thread_pool_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_backup_thread_pool_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup Thread Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 415, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_backup_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, error) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{error}}", "metric": "", "query": "sum(delta(\n tikv_backup_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, error) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 416, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Write CF SST Size", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 417, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Default CF SST Size", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 418, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total", "metric": "", "query": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{cf}}", "metric": "", "query": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, cf) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup SST Generation Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 419, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Scan SST Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 420, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Scan SST Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 421, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"save.*\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"save.*\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Save SST Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 422, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-99.9%", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum((\n tikv_backup_range_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) / sum((\n tikv_backup_range_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-avg", "metric": "", "query": "(sum((\n tikv_backup_range_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) / sum((\n tikv_backup_range_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup SST Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 423, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "External Storage Create Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 424, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_external_storage_create_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_external_storage_create_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "External Storage Create Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 425, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Checksum Request Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 426, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, device) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{device}}", "metric": "", "query": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, device) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IO Utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 427, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "import-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, tid) > 0", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "import-{{instance}}-{{tid}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, tid) > 0", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "import-count-{{instance}}", "metric": "", "query": "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import CPU Utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 428, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Thread Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 429, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_import_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, error, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{error}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_import_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, error, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 430, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{request}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{request}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_import_rpc_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request) / sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{request}}", "metric": "", "query": "(sum(rate(\n tikv_import_rpc_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request) / sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{request}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 431, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{request}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Ops", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 432, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=~\"download|write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=~\"download|write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Write/Download RPC Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 433, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"queue\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"queue\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Wait Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 434, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"read\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"read\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Read SST Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 435, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"rewrite\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"rewrite\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Rewrite SST Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 56 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 436, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Ingest RPC Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 56 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 437, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_ingest_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_ingest_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Ingest SST Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 56 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 438, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_ingest_byte_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_import_ingest_byte_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Ingest SST Bytes", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 439, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total", "metric": "", "query": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Download SST Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 440, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_import_local_write_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_import_local_write_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Local Write keys", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 441, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_local_write_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_local_write_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Local Write bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 442, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "sum", "metric": "", "query": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL Expired", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 443, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_cloud_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cloud, req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cloud}}-{{req}}", "metric": "", "query": "sum(rate(\n tikv_cloud_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cloud, req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "cloud request", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Backup & Import", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 444, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total number of encryption data keys in use", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 445, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_encryption_data_key_storage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_encryption_data_key_storage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encryption data keys", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of files being encrypted", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 446, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_encryption_file_num\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_encryption_file_num\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encrypted files", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Flag to indicate if encryption is initialized", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 447, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_encryption_is_initialized\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_encryption_is_initialized\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encryption initialized", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total size of encryption meta files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 448, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_encryption_meta_file_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{instance}}", "metric": "", "query": "((\n tikv_encryption_meta_file_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encryption meta files size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 449, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"encrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "encrypt-{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"encrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"decrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "decrypt-{{req}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"decrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encrypt/decrypt data nanos", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Writing or reading file duration (second)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 450, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_encryption_write_read_file_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg", "metric": "", "query": "(sum(rate(\n tikv_encryption_write_read_file_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) / sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count", "metric": "", "query": "sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read/write encryption meta duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Encryption", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 451, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": [ { "options": { "0": { "color": "red", "index": null, "text": "Disabled" }, "1": { "color": "green", "index": null, "text": "Enabled" } }, "type": "value" } ], "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 452, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_log_backup_enabled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "((\n tikv_log_backup_enabled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Endpoint Status", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": [ { "options": { "0": { "color": "green", "index": null, "text": "Running" }, "1": { "color": "yellow", "index": null, "text": "Paused" }, "2": { "color": "red", "index": null, "text": "Error" } }, "type": "value" } ], "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 453, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "min((\n tikv_log_backup_task_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "min((\n tikv_log_backup_task_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Task Status", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 454, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "name" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "tidb_log_backup_advancer_owner > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "tidb_log_backup_advancer_owner > 0", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Advancer Owner", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average flush size of last 30mins.", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 455, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(increase(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) / sum(increase(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [30m]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "(sum(increase(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) / sum(increase(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [30m]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Average Flush Size", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The current total flushed file number of this run.", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": 0, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 456, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) > 0", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flushed Files (Last 30m) Per Host", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": 0, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 457, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [30m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [30m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flush Times (Last 30m)", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "This is the summary of the size has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 458, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Total Flushed Size (Last 30m)", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": 0, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 459, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flush Files (Last 30m)", "transformations": [], "transparent": false, "type": "stat" }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of log backup threads. \n**(Note this is the average usage for a period of time, some peak of CPU usage may be lost.)**", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 460, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 461, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_handle_kv_batch_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_handle_kv_batch_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Handle Event Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The data rate of initial scanning emitting events.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 462, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_incremental_scan_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_incremental_scan_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scan Generate Event Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 463, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(time() * 1000 - max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (task) / 262144 > 0)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ task }}", "metric": "", "query": "(time() * 1000 - max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (task) / 262144 > 0)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Abnormal Checkpoint TS Lag", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The estimated memory usage by the streaming backup module.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 464, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_log_backup_heap_memory\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_log_backup_heap_memory\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Of Events", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 465, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-total", "metric": "", "query": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Observed Region Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The errors met when backing up.\n**They are retryable, don't worry.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 466, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The errors met when backing up.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 467, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_fatal_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_fatal_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Fatal Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 468, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Current Time", "bars": false, "dashes": true, "fill": 0, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (task) / 262144 > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{task}}", "metric": "", "query": "max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (task) / 262144 > 0", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "time() * 1000", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Current Time", "metric": "", "query": "time() * 1000", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Checkpoint TS of Tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "dateTimeAsIsoNoDateIfToday", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of flushing a batch of file.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 469, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_flush_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_flush_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flush Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of scanning the initial data from local DB and transform them into apply events.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 470, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_initial_scan_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_initial_scan_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Initial scanning duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of converting a raft request into a apply event.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 471, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"to_stream_event\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"to_stream_event\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Convert Raft Event duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of waiting the mutex of the controller.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 472, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"get_router_lock\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"get_router_lock\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Wait for Lock Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The number of KV-modify of each raft command observed.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 473, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_handle_kv_batch_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_handle_kv_batch_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Command Batch Size", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The total cost of saving an event into temporary file.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 474, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_to_temp_file\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_to_temp_file\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Save to Temp File Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The total cost of writing a event into temporary file.\nComparing to the ***Save*** duration, it doesn't contain the time cost of routing the task by range / task.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 475, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=\"write_to_tempfile\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=\"write_to_tempfile\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Write to Temp File Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of collecting metadata and call the UNIX system call *write* for each event.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 476, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=\"syscall_write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=\"syscall_write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "System Write Call Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal message type count.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 477, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_interal_actor_acting_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{message}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_interal_actor_acting_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Internal Message Type", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 478, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_log_backup_interal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{message}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_log_backup_interal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Internal Message Handling Duration (P99)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 479, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9,(\n sum(rate(\n tikv_log_backup_interal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{message}}", "metric": "", "query": "histogram_quantile(0.9,(\n sum(rate(\n tikv_log_backup_interal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Internal Message Handling Duration (P90)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal read throughput of RocksDB during initial scanning. This panel can roughly present the read through to the hard disk of initial scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 480, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scan RocksDB Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Misc statistics of RocksDB during initial scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 481, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op!~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf, op) > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{op}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op!~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf, op) > 0", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scan RocksDB Operation", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The reason of triggering initial scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 482, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_initial_scan_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{reason}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_initial_scan_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scanning Trigger Reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 483, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_metadata_key_operation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_metadata_key_operation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Checkpoint Key Putting", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 484, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_advancer_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"checkpoint\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_advancer_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"checkpoint\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Request Checkpoint Batch Size", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 485, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"tick\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"tick\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Tick Duration", "tooltip": { "msResolution": true, "shared": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The reason of advancer failed to be advanced.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 486, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_region_request_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason!=\"retryable-scan-region\"}\n [$__rate_interval]\n)) by (reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{reason}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_region_request_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason!=\"retryable-scan-region\"}\n [$__rate_interval]\n)) by (reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Checkpoint Failure Reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The result of getting region checkpoints.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 487, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_region_request\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (result) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{result}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_region_request\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (result) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request Result", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 488, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ step }}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Tick Duration (P99)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 489, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ step }}", "metric": "", "query": "histogram_quantile(0.9,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Tick Duration (P90)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The frequent of getting region level checkpoint.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 490, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"get-regions-in-range\"}\n [$__rate_interval]\n)) by (step, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{step}}-{{instance}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"get-regions-in-range\"}\n [$__rate_interval]\n)) by (step, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Get Region Operation Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The variant of checkpoint group.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 491, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"try-advance\"}\n [$__rate_interval]\n)) by (step, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{step}}-{{instance}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"try-advance\"}\n [$__rate_interval]\n)) by (step, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Try Advance Trigger Time", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Backup Log", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 492, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The changing trend of the slowness on I/O operations. 'value > 0' means the related store might have a slow trend.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 493, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_slow_trend\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_slow_trend\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Slow Trend", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The changing trend of QPS on each store. 'value < 0' means the QPS has a dropping trend.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 494, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_slow_trend_result\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_slow_trend_result\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "QPS Changing Trend", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The sampling latency of recent queries. A larger value indicates that the store is more likely to be the slowest store.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 495, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_slow_trend_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_slow_trend_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "AVG Sampling Latency", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The QPS of each store.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 496, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_slow_trend_result_value\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_slow_trend_result_value\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "QPS of each store", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Slow Trend Statistics", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 497, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The 99 quantile durtion of status server API requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 498, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{path}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{path}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_status_server_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path) / sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{path}}", "metric": "", "query": "(sum(rate(\n tikv_status_server_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path) / sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{path}}", "metric": "", "query": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Status API Request Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 499, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{path}}", "metric": "", "query": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Status API Request (op/s)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Status Server", "transformations": [], "transparent": false, "type": "row" } ], "refresh": "1m", "rows": [], "schemaVersion": 14, "sharedCrosshair": false, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "k8s_cluster", "multi": false, "name": "k8s_cluster", "options": [], "query": "label_values(tikv_engine_block_cache_size_bytes, k8s_cluster)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", "options": [], "query": "label_values(tikv_engine_block_cache_size_bytes{k8s_cluster =\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "db", "multi": true, "name": "db", "options": [], "query": "label_values(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "command", "multi": true, "name": "command", "options": [], "query": "query_result(tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"} != 0)", "refresh": 2, "regex": "/\\btype=\"([^\"]+)\"/", "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": ".*", "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "instance", "multi": false, "name": "instance", "options": [], "query": "label_values(tikv_engine_size_bytes{k8s_cluster =\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": true, "label": "titan_db", "multi": true, "name": "titan_db", "options": [], "query": "label_values(tikv_engine_titandb_num_live_blob_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "hidden": false, "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Test-Cluster-TiKV-Details", "uid": "RDVQiEzZz", "version": 0 }