{ "__inputs": [ { "description": "", "label": "test-cluster", "name": "DS_TEST-CLUSTER", "pluginId": "prometheus", "pluginName": "Prometheus", "type": "datasource" } ], "annotations": { "list": [] }, "description": "", "editable": true, "gnetId": null, "graphTooltip": 1, "hideControls": false, "id": null, "links": [], "panels": [ { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 1, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Write Pipeline Composition", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 2, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Write Raft Log .99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Propose Wait .99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Apply Wait .99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Replicate Raft Log .99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Apply Duration .99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Pipeline Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Read Duration Composition", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 3, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Get Snapshot .99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Cop Wait .99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.95,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Cop Handle .99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.95,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cop Read Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Duration", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 4, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage size per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 5, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type = \"used\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type = \"used\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The available capacity size of each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 6, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"available\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"available\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Available size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The capacity size per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 7, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Capacity size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU usage of each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 8, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "quota-{{instance}}", "metric": "", "query": "((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memory usage per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 9, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "quota-{{instance}}", "metric": "", "query": "((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The I/O utilization per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 10, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{device}}", "metric": "", "query": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IO utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total bytes of read and write in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 11, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-write", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-read", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-in-memory-engine-read", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "MBps", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of leaders on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 12, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of the gRPC message failures", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 13, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-grpc-msg-fail", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"noop\"}\n [$__rate_interval]\n)) by (instance) < 1", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-pd-heartbeat", "metric": "", "query": "sum(delta(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"noop\"}\n [$__rate_interval]\n)) by (instance) < 1", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Errps", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of leaders on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 14, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Leader", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Regions and Buckets on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 15, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"buckets\"}\n \n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-buckets", "metric": "", "query": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"buckets\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "TiKV uptime since the last restart", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 16, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(time() - ((\n process_start_time_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(time() - ((\n process_start_time_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",job=~\".*tikv\"}\n \n)) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Uptime", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Cluster", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 17, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.0, "yaxis": "left" } ] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 18, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.0, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Critical error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "\nIndicates occurrences of events that make the TiKV instance unavailable\ntemporarily, such as Write Stall, Channel Full, Scheduler Busy, and Coprocessor\nFull", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 19, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "scheduler-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "channelfull-{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"full\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "coprocessor-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"full\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write_stall_percentile99\",db=~\"$db\"}\n \n)) by (instance, db) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "stall-{{instance}}-{{db}}", "metric": "", "query": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write_stall_percentile99\",db=~\"$db\"}\n \n)) by (instance, db) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "store-write-channelfull-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_process_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n tikv_raftstore_process_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Server is busy", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of reporting failure messages", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 20, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_report_failure_msg_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance, store_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}-to-{{store_id}}", "metric": "", "query": "sum(rate(\n tikv_server_report_failure_msg_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance, store_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Server report failures", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of different raftstore errors on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 21, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status!~\"success|all\"}\n [$__rate_interval]\n)) by (instance, status) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{status}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status!~\"success|all\"}\n [$__rate_interval]\n)) by (instance, status) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raftstore error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of scheduler errors per type on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 22, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"snapshot_err|prepare_write_err\"}\n [$__rate_interval]\n)) by (instance, stage) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{stage}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"snapshot_err|prepare_write_err\"}\n [$__rate_interval]\n)) by (instance, stage) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of different coprocessor errors on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 23, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of gRPC message errors per type on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 24, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC message error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of dropped leaders per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 25, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"leader\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Leader drop", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of missing leaders per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 26, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_leader_missing\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_leader_missing\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Leader missing", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "RocksDB damaged SST files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 27, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_rocksdb_damaged_files\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-existed", "metric": "", "query": "((\n tikv_rocksdb_damaged_files\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_rocksdb_damaged_files_deleted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-deleted", "metric": "", "query": "((\n tikv_rocksdb_damaged_files_deleted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Damaged files", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of Log Replication Reject caused by follower memory insufficient", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 28, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_append_rejects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_append_rejects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Log Replication Rejected", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Errors", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 29, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The size of each column family", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 30, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_engine_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CF size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of channel full errors on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 31, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Channel full", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of leaders being written on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 32, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Active written leaders", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 33, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Approximate region size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The approximate Region size", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 34, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_region_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_region_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Approximate region size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 35, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_region_written_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_region_written_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region written bytes", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average rate of writing bytes to Regions per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 36, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_region_written_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_region_written_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region average written bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 37, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_region_written_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_region_written_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region written keys", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average rate of written keys to Regions per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 38, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_region_written_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_region_written_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region average written keys", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Bucketed histogram of clear overlap region duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 39, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Clear overlap region duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The key size for every put of apply worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 40, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_apply_key_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_apply_key_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Written key size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 41, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_apply_value_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_apply_value_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Written value size for every put of apply worker", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of peers in hibernated state", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 42, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_hibernated_peer_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, state) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{state}}", "metric": "", "query": "sum((\n tikv_raftstore_hibernated_peer_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, state) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Hibernate Peers", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 43, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{name}}", "metric": "", "query": "((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n raft_engine_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-raft-engine", "metric": "", "query": "((\n raft_engine_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory trace", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 44, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raft_entries_evict_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raft_entries_evict_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Entry Cache Evicts", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 45, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_address_resolve_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_address_resolve_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Resolve address duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 46, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{priority}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "99% Thread Pool Schedule Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average rate of written keys to Regions per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 47, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{priority}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Average Thread Pool Schedule Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 48, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "copr-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk IO time per second", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ns", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 49, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "copr-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk IO bytes per second", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ns", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Server", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 50, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different kinds of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 51, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{priority}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC message count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different kinds of gRPC message which is failed", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 52, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC message failed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The 99% percentile of execution time of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 53, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{priority}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% gRPC message duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average execution time of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 54, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{priority}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Average gRPC message duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The 99% percentile of execution time of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 55, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_grpc_req_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% request {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_grpc_req_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_grpc_resp_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% response {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_grpc_resp_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_grpc_req_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_req_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg request {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_server_grpc_req_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_req_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_grpc_resp_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_resp_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg response {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_server_grpc_resp_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_resp_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_request_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% kv get batch {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_request_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_request_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_request_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg kv batch {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_server_request_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_request_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC batch size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 56, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_message_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_message_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_raft_message_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_raft_message_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_server_raft_message_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_raft_message_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "raft message batch size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The QPS of different sources of gRPC request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 57, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_request_source_counter_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{source}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_request_source_counter_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC request sources QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of different sources of gRPC request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 58, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_request_source_duration_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{source}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_request_source_duration_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC request sources duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The QPS of different resource groups of gRPC request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 59, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_resource_group_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_resource_group_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC resource group QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "gRPC", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 60, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total count of different kinds of commands received", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 61, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_command_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_command_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage command total", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of engine asynchronous request errors", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 62, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status!~\"all|success\"}\n [$__rate_interval]\n)) by (status, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{status}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status!~\"all|success\"}\n [$__rate_interval]\n)) by (status, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async request error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by processing asynchronous write requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 63, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Storage async write duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async write duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 64, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by processing asynchronous snapshot requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 65, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async snapshot duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 66, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async snapshot duration without the involving of raftstore", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 67, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration (pure local read)", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async snapshot duration without the involving of raftstore", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 68, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration (pure local read)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Read index propose wait duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 69, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Read index propose wait duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Read index propose wait duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 70, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read index propose wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Read index confirm duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 71, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Read index confirm duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Read index confirm duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 72, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read index confirm duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "CPU usage measured over a 30 second window", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 73, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_storage_process_stat_cpu_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_storage_process_stat_cpu_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Process Stat Cpu Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 74, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_full_compact_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_full_compact_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_full_compact_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Full compaction duration seconds", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 75, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_full_compact_pause_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Full compaction pause duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 76, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_full_compact_increment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Full compaction per-increment duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Storage", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 77, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 78, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/.*-total/", "bars": false, "fill": 1, "fillBelowTo": null, "lines": true, "yaxis": 2, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_local_read_reject_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-reject-by-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_local_read_reject_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_local_read_executed_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-total", "metric": "", "query": "sum(rate(\n tikv_raftstore_local_read_executed_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_local_read_executed_stale_read_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-stale-read", "metric": "", "query": "sum(rate(\n tikv_raftstore_local_read_executed_stale_read_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Local reader requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Local Reader", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 79, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of raftstore thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 80, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(raftstore|rs)_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft store CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of async apply", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 81, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"apply_[0-9]+\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Async apply CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of store writer thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.8, "yaxis": "left" } ] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 82, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_write.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.8, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Store writer CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of gRPC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 83, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"grpc.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC poll CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of scheduler worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 3.6, "yaxis": "left" } ] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 84, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 3.6, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Scheduler worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of the unified read pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 85, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified_read_po.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified_read_po.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Unified read pool CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of RocksDB", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 86, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"rocksdb.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"rocksdb.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "RocksDB CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 87, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"gc_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"gc_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 88, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"region_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"region_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 89, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"snap_generator.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"snap_generator.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snap generator CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 90, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"background.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"background.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Background worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 91, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog_fetch.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog_fetch.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raftlog fetch worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 92, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 93, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(backup-worker|bkwkr|backup_endpoint).*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"(backup-worker|bkwkr|backup_endpoint).*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 94, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdcwkr.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-worker", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdcwkr.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-tso", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdc_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-endpoint", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cdc_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CDC worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of raftstore thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 95, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso_worker\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"tso_worker\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TSO worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of storage read pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 3.6, "yaxis": "left" } ] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 96, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_norm.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-normal", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_norm.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_high.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-high", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_high.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_low.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-low", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"store_read_low.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 3.6, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Storage read pool CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of coprocessor read pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 97, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_normal.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-normal", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_normal.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_high.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-high", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_high.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_low.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-low", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"cop_low.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Coprocessor read pool CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of IME threads", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 98, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{name}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IME CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 99, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name!~\"rocksdb.*\"}\n [$__rate_interval]\n)) by (instance, name) > 0.8\n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{instance}}", "metric": "", "query": "topk(20,(\n sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name!~\"rocksdb.*\"}\n [$__rate_interval]\n)) by (instance, name) > 0.8\n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Busy Threads (>80%)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Thread CPU", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 100, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The throughput of disk write per IO type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 101, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write IO bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The throughput of disk read per IO type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 102, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=\"read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read IO bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The threshold of disk IOs per priority", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 103, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_rate_limiter_max_bytes_per_sec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_rate_limiter_max_bytes_per_sec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IO threshold", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "IO rate limiter request wait duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 104, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Rate Limiter Request Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "IO Breakdown", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 105, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by processing asynchronous write requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 106, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The store time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 107, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The apply time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 108, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_apply_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_apply_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The propose wait time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 109, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store propose wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The batch wait time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 110, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store batch wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The send-to-write-queue time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 111, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store send to write queue duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The send raft message of the proposal duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 112, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store send proposal duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The write kv db end duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 113, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_write_kvdb_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store write kv db end duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The before write time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 114, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store before write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The persist duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 115, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store persist duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The write end duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 116, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store write end duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The commit but not persist duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 117, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store commit but not persist duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The commit and persist duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 118, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store commit and persist duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Waterfall", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 119, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed for peer processes to be ready in Raft", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 120, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Process ready duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed for peer processes to be ready in Raft", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 121, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_raft_process_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_raft_process_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Process ready duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time duration of store write loop when store-io-pool-size is not zero.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 122, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write loop duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time duration of store write loop on each TiKV instance when store-io-pool-size is not zero.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 123, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_loop_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Store write loop duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft appends log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 124, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Append log duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft appends log on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 125, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_append_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_append_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_append_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_append_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_append_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_append_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Append log duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft commits log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 126, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Commit log duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft commits log on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 127, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Commit log duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft applies log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 128, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply log duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed for Raft to apply logs per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 129, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_apply_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_apply_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Apply log duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed for Raft Client wait connection ready", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 130, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Raft Client Wait Connection Ready Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed for Raft Client wait connection ready per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 131, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{to}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{to}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_client_wait_ready_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_raft_client_wait_ready_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) / sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{to}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_server_raft_client_wait_ready_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) / sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{to}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_client_wait_ready_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (to, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Raft Client Wait Connection Ready Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The throughput of disk write per IO type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 132, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_io_reschedule_region_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "rechedule-{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_io_reschedule_region_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_io_reschedule_pending_tasks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "pending-task-{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_io_reschedule_pending_tasks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store io task reschedule", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when store write task block on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 133, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Write task block duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft IO", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 134, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The proposal count of a Regions in a tick", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 135, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_proposal_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_proposal_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft proposals per ready", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of proposals per type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 136, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|normal|read_index\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|normal|read_index\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft read/write proposals", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of read proposals which are made by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 137, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|read_index\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"local_read|read_index\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft read proposals per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of write proposals which are made by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 138, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"normal\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"normal\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft write proposals per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each proposal", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 139, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Propose wait duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each proposal in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 140, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Propose wait duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each store write task", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 141, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write wait duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each store write task in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 142, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Store write wait duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each apply task", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 143, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply wait duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each apply task in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 144, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Apply wait duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The handle duration of each store write task msg", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 145, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_store_write_handle_msg_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_store_write_handle_msg_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write handle msg duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The distribution of write trigger size", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 146, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_store_write_trigger_wb_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_store_write_trigger_wb_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write trigger size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate at which peers propose logs", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 147, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_propose_log_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_propose_log_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft propose speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate at which peers propose logs", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 148, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "store-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "apply-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Perf Context duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Propose", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 149, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different ready type of Raft", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 150, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_ready_handled_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_ready_handled_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"ready\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Ready handled", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The max time consumed by raftstore events", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 151, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_raftstore_event_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_raftstore_event_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_broadcast_normal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "broadcast_normal {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_broadcast_normal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max duration of raft store events", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Replica read lock checking duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 152, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_replica_read_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_replica_read_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Replica read lock checking duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of fsm reschedule ops", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 153, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_batch_system_fsm_reschedule_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_batch_system_fsm_reschedule_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Fsm reschedule ops", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of store fsm waiting to be polled", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 154, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_batch_system_fsm_schedule_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_batch_system_fsm_schedule_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store fsm schedule wait duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of apply fsm waiting to be polled.e", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 155, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_batch_system_fsm_schedule_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_batch_system_fsm_schedule_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply fsm schedule wait duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Total time for an store FSM to finish processing all messages, potentially over multiple polling rounds.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 156, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_batch_system_fsm_poll_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_batch_system_fsm_poll_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store fsm poll duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Total time for an apply FSM to finish processing all messages, potentially over multiple polling rounds", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 157, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_batch_system_fsm_poll_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_batch_system_fsm_poll_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply fsm poll duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Number of polling rounds for an store FSM to finish processing all messages", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 158, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_batch_system_fsm_poll_rounds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_batch_system_fsm_poll_rounds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store fsm poll round", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Number of polling rounds for an apply FSM to finish processing all messages", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 159, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_batch_system_fsm_poll_rounds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_batch_system_fsm_poll_rounds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply fsm poll round", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Number of store fsm polled in one poll", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 160, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_batch_system_fsm_count_per_poll_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_batch_system_fsm_count_per_poll_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"store\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store fsm count per poll", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Number of apply fsm polled in one poll", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 161, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_batch_system_fsm_count_per_poll_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_batch_system_fsm_count_per_poll_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply fsm count per poll", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The length of peer msgs for each round handling", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 162, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_peer_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_peer_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Peer msg length distribution", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The length of apply msgs for each round handling", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 163, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_apply_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_apply_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply msg length distribution", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of regions that enable apply unpersisted raft log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 164, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_raft_enable_unpersisted_apply_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_raft_enable_unpersisted_apply_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Enable apply unpersisted log region count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of raft logs between apply and persisted index", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 165, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-max", "metric": "", "query": "histogram_quantile(1,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply ahead of persistence raft log count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Process", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 166, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Raft messages sent by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 167, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Sent messages per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Raft messages flushed by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 168, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_message_flush_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_message_flush_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flush messages per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Raft messages received by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 169, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_message_recv_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_message_recv_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Receive messages per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of different types of Raft messages that are sent", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 170, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status=\"accept\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status=\"accept\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Messages", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of vote messages that are sent in Raft", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 171, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"vote\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"vote\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Vote", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of dropped Raft messages per type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 172, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_dropped_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_dropped_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status=\"drop\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",status=\"drop\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft dropped messages", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Message", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 173, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of admin proposals", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 174, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"conf_change|transfer_leader\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"conf_change|transfer_leader\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Admin proposals", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of the processed apply command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 175, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_admin_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"compact\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_admin_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"compact\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Admin apply", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of raftstore split checks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 176, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_check_split_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"ignore\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_check_split_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"ignore\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Check split", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when running split check in .9999", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 177, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_check_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_check_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% Check split duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 178, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_load_base_split_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(delta(\n tikv_load_base_split_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Load base split event", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 179, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.8,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "80%-{{instance}}", "metric": "", "query": "histogram_quantile(0.8,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_load_base_split_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_load_base_split_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_load_base_split_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_load_base_split_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Load base split duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 180, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_peer_in_flashback_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_peer_in_flashback_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Peer in Flashback State", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Admin", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 181, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 182, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log GC write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 183, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_raft_log_kv_sync_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log GC kv sync duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 184, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_log_gc_write_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log GC write operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 185, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_log_gc_seek_operations_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_log_gc_seek_operations_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log GC seek operations ", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 186, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_log_lag_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_log_lag_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log lag", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 187, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_log_gc_skipped\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_log_gc_skipped\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log gc skipped", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 188, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_log_gc_failed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_log_gc_failed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log GC failed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 189, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_entry_fetches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_entry_fetches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log fetch ", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 190, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/pending-task/", "bars": false, "fill": 1, "fillBelowTo": null, "lines": true, "transform": "negative-Y", "yaxis": 2, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_entry_fetches_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog-fetch-worker\"}\n \n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "pending-task {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftlog-fetch-worker\"}\n \n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log async fetch task duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Log", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 191, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of operations per second", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 192, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_write_apply_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_write_apply_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_read_entry_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read_entry {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_read_entry_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_read_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read_message {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_read_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Operation", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time used in write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 193, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n raft_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n raft_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The I/O flow rate", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 194, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_background_rewrite_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "rewrite-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_background_rewrite_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "99% duration breakdown of write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 195, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_preprocess_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "wait {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_preprocess_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "wal {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "apply {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Duration Breakdown (99%)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The bytes per write", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 196, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n raft_engine_write_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_write_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Bytes / Written", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "999% duration breakdown of WAL write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 197, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_write_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_sync_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "sync {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_sync_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_allocate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "allocate {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_allocate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_rotate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "rotate {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_rotate_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "WAL Duration Breakdown (999%)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average number of files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 198, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_log_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "avg((\n raft_engine_log_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_swap_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "swap {{$additional_groupby}}", "metric": "", "query": "avg((\n raft_engine_swap_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_recycled_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-recycle {{$additional_groupby}}", "metric": "", "query": "avg((\n raft_engine_recycled_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "File Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The 99% duration of operations other than write", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 199, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_entry_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read_entry {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_entry_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read_message {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_read_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_purge_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "purge {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_purge_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Other Durations (99%)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average number of log entries", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 200, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_log_entry_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "avg((\n raft_engine_log_entry_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Entry Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The compression ratio per write", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 201, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_write_compression_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n raft_engine_write_compression_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n raft_engine_write_compression_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_write_compression_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Compression Ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Engine", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 202, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of get operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 203, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "memtable {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"block_cache_data_hit|block_cache_filter_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "block_cache {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"block_cache_data_hit|block_cache_filter_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l0\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "l0 {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l0\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l1\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "l1 {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l1\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l2_and_up\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "l2_and_up {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_get_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_hit_l2_and_up\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Get operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing get operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 204, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_get_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"get_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Get duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of seek operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 205, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seek {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seek_found {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_seek_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "next {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "next_found {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_next_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "prev {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "prev_found {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"number_db_prev_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Seek operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing seek operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 206, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_seek_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"seek_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Seek duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of write operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 207, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"write_done_by_self|write_done_by_other\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "done {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=~\"write_done_by_self|write_done_by_other\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_timeout\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "timeout {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_timeout\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_with_wal\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "with_wal {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_write_served\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_with_wal\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 208, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_write_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of WAL sync operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 209, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_wal_file_synced\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "sync {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_wal_file_synced\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "WAL sync operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing write wal operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 210, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_write_wal_time_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_wal_micros_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write WAL duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of compaction and flush operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 211, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_event_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_event_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing WAL sync operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 212, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_wal_file_sync_micro_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_sync_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "WAL sync duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Compaction guard actions", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 213, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_compaction_guard_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=~\"default|write\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{ type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_compaction_guard_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=~\"default|write\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction guard actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing the compaction and flush operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 214, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_compaction_time\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"compaction_time_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "How many sst files are compacted in a compaction job", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 215, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_num_files_in_single_compaction\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"num_files_in_single_compaction_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Job Size(files)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when reading SST files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 216, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_sst_read_micros\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"sst_read_micros_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "SST read duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 217, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_compaction_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (cf, reason, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{reason}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_compaction_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (cf, reason, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The block cache size. Broken down by column family if shared block cache is disabled.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 218, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n avg((\n tikv_engine_block_cache_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, instance) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{cf}}", "metric": "", "query": "topk(20,(\n avg((\n tikv_engine_block_cache_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, instance) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block cache size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The hit rate of memtable", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 219, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "hit {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_memtable_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"memtable_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memtable hit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The flow of different kinds of block cache operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 220, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total_read {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total_written {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_byte_write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_bytes_insert\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "data_insert {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_bytes_insert\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_insert\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "filter_insert {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_insert\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_evict\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "filter_evict {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_bytes_evict\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_insert\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "index_insert {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_insert\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_evict\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "index_evict {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_bytes_evict\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block cache flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The hit rate of block cache", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 221, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "all {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "data {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "filter {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "index {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) ) / ((sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) ) + (sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filter_match\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filter_match\"}\n [$__rate_interval]\n)) by ($additional_groupby) )))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "bloom prefix {{$additional_groupby}}", "metric": "", "query": "((sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) ) / ((sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filtered\"}\n [$__rate_interval]\n)) by ($additional_groupby) ) + (sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"last_level_seek_filter_match\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_bloom_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"non_last_level_seek_filter_match\"}\n [$__rate_interval]\n)) by ($additional_groupby) )))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block cache hit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The flow of different kinds of operations on keys", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 222, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_written\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "written {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"keys_written\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_compaction_num_corrupt_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "corrupt {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_compaction_num_corrupt_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keys flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different kinds of block cache operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 223, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total_add {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "data_add {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_data_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "filter_add {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_filter_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "index_add {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_index_add\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add_failures\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "add_failures {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"block_cache_add_failures\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block cache operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The flow rate of read operations per type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 224, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "get {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"iter_bytes_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "scan {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"iter_bytes_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of keys in each column family", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 225, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_estimate_num_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_engine_estimate_num_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total keys", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The flow of different kinds of write operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 226, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "wal {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The bytes per read", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 227, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_bytes_per_read\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_read_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Bytes / Read", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The flow rate of compaction operations per type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 228, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_compaction_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_compaction_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_compaction_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "written {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_compaction_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_written\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"flush_write_bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "flushed {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"flush_write_bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The bytes per write", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 229, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_bytes_per_write\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"bytes_per_write_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Bytes / Write", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The read amplification per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 230, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_read_amp_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_total_read_bytes\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_engine_read_amp_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_estimate_useful_bytes\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_engine_read_amp_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_total_read_bytes\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_engine_read_amp_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"read_amp_estimate_useful_bytes\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read amplification", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The pending bytes to be compacted", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 231, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction pending bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of snapshot of each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 232, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_engine_num_snapshots\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_engine_num_snapshots\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Number of snapshots", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The compression ratio of each level", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 233, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_compression_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, level, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-L{{level}} {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_compression_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, level, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compression ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of SST files for different column families in each level", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 105 }, "height": null, "hideTimeOverride": false, "id": 234, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_num_files_at_level\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, level, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-L{{level}} {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_num_files_at_level\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) by (cf, level, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Number files at each level", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time that the oldest unreleased snapshot survivals", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 105 }, "height": null, "hideTimeOverride": false, "id": 235, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_engine_oldest_snapshot_duration\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_engine_oldest_snapshot_duration\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Oldest snapshots duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The level that the external file ingests into", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 112 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 236, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_engine_ingestion_picked_level_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_engine_ingestion_picked_level_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Ingestion picked level", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Bucketed histogram of ingest external SST files duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 112 }, "height": null, "hideTimeOverride": false, "id": 237, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_ingest_external_file_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{cf}}-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_ingest_external_file_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_ingest_external_file_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{cf}}-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_ingest_external_file_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_ingest_external_file_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) / sum(rate(\n tikv_storage_ingest_external_file_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{cf}}-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_ingest_external_file_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) / sum(rate(\n tikv_storage_ingest_external_file_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_ingest_external_file_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{cf}}-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_ingest_external_file_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cf, type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Ingest SST duration seconds", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 119 }, "height": null, "hideTimeOverride": false, "id": 238, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_write_stall_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_write_stall_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Stall Reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time which is caused by write stall", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 119 }, "height": null, "hideTimeOverride": false, "id": 239, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"write_stall_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write stall duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Stall conditions changed of each column family", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 126 }, "height": null, "hideTimeOverride": false, "id": 240, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_engine_stall_conditions_changed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{cf}}-{{type}}", "metric": "", "query": "((\n tikv_engine_stall_conditions_changed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Stall conditions changed of each CF", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memtable size of each column family", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 126 }, "height": null, "hideTimeOverride": false, "id": 241, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"mem-tables-all\"}\n \n)) by (cf, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}} {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$db\",type=\"mem-tables-all\"}\n \n)) by (cf, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memtable size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": "db", "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "RocksDB - $db", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 242, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 243, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_titandb_num_live_blob_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "live blob file num {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_engine_titandb_num_live_blob_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_titandb_num_obsolete_blob_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "obsolete blob file num {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_engine_titandb_num_obsolete_blob_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 244, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_titandb_live_blob_file_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "live blob file size {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_engine_titandb_live_blob_file_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_titandb_obsolete_blob_file_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "obsolete blob file size {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_engine_titandb_obsolete_blob_file_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The blob cache size.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 245, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n avg((\n tikv_engine_blob_cache_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (cf, instance) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{cf}}", "metric": "", "query": "topk(20,(\n avg((\n tikv_engine_blob_cache_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (cf, instance) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob cache size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The hit rate of block cache", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 246, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "all {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) / (sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_hit\"}\n [$__rate_interval]\n)) by ($additional_groupby) + sum(rate(\n tikv_engine_blob_cache_efficiency\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_cache_miss\"}\n [$__rate_interval]\n)) by ($additional_groupby) ))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob cache hit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 247, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_iter_touch_blob_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_iter_touch_blob_file_count_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Iter touched blob file count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 248, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_key_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_key_size_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob key size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 249, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_value_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_value_size_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob value size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 250, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_get\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "get {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_get\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob get operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 251, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%-{{type}} {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_get_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob get duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 252, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_titandb_blob_file_discardable_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (ratio, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ratio}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_engine_titandb_blob_file_discardable_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n \n)) by (ratio, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file discardable ratio distribution", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 253, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_seek\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seek {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_seek\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_prev\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "prev {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_prev\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_next\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "next {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"number_blob_next\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob iter operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 254, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_seek_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob seek duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 255, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_next_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob next duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 256, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_average\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95%-{{type}} {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile95\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_percentile99\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_prev_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\".*_max\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob prev duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 257, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob keys flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 258, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_average\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_average\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_max\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_file_read_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_read_micros_max\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file read duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 259, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob bytes flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 260, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_average\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_average\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_max\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_file_write_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_write_micros_max\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 261, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_file_synced\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "sync {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_file_synced\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file sync operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 262, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_gc_action_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_gc_action_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC action", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 263, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_average\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_average\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_max\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_file_sync_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_file_sync_micros_max\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob file sync duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 264, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_average\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_average\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile95\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_percentile99\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_max\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_gc_micros_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_micros_max\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 265, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_gc_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_gc_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"keys.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC keys flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 266, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_gc_input_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_input_file_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC input file size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 267, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_gc_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_gc_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=~\"bytes.*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC bytes flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 268, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_average\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_average\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile95\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile95\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile99\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_percentile99\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_max\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_engine_blob_gc_output_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\",type=\"blob_gc_output_file_max\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC output file size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 269, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_blob_gc_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_engine_blob_gc_file_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"$titan_db\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob GC file count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": "titan_db", "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Titan - $titan_db", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 270, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Operation per second for cf", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 271, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_kv_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_kv_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total bytes of read in RocksDB and in-memory engine(the same with panel Cluster/MBps for read)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 272, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "rocksdb-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "in-memory-engine-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read MBps", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when handling coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 273, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor Handle duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Count of region cache hit", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 274, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Cache Hit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Region cache hit rate", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 275, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "rate-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_memory_engine\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_snapshot_type_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Cache Hit Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Reasons for region cache miss", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 276, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_snapshot_acquire_failed_reason_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Cache Miss Reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memory usage of the in-memory engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 277, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_in_memory_engine_memory_usage_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "avg((\n tikv_in_memory_engine_memory_usage_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different types of region", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 278, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_in_memory_engine_cache_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}--{{type}}", "metric": "", "query": "avg((\n tikv_in_memory_engine_cache_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rang cache engine garbage collection information", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 279, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_gc_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_gc_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC Filter", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The handle duration of region gc", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 280, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_in_memory_engine_gc_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_in_memory_engine_gc_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region GC Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The handle duration of region load", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 281, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_in_memory_engine_load_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_in_memory_engine_load_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region Load Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of region loading per seconds", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 282, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_in_memory_engine_load_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_in_memory_engine_load_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Load Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The handle duration of region eviction", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 283, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_in_memory_engine_eviction_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_in_memory_engine_eviction_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region Eviction Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of region eviction per seconds", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 284, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_in_memory_engine_eviction_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(delta(\n tikv_in_memory_engine_eviction_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Eviction Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed of write in region cache engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 285, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Write duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed of write in region cache engine per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 286, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_in_memory_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_in_memory_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_in_memory_engine_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% In-memory engine write duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed of prepare for write in the in-memory engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 287, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Prepare for write duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed of prepare for write in the in-memory engine per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 288, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_prepare_for_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% In-memory engine prepare for write duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different type of iteration operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 289, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_seek\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seek {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_seek\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_seek_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seek_found {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_seek_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_next\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "next {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_next\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_next_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "next_found {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_next_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_prev\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "prev {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_prev\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_prev_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "prev_found {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_locate\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"number_db_prev_found\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Iterator operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing seek operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 290, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.95,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.95,(\n sum(rate(\n tikv_in_memory_engine_seek_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_in_memory_engine_seek_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_seek_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_in_memory_engine_seek_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_in_memory_engine_seek_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Seek duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the oldest one", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 291, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Oldest Auto GC SafePoint", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Unlike the auto gc safe point used for TiKV, the safe point for in-memory engine is per region and this is the newest one", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 292, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Newest Auto GC SafePoint", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The gap between newest auto gc safe point and oldest auto gc safe point of regions cached in the in-memroy engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 293, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18) - sum((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum((\n tikv_in_memory_engine_newest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18) - sum((\n tikv_in_memory_engine_oldest_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Auto GC SafePoint Gap", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The gap between tikv auto gc safe point and in-memory engine oldest auto gc safe point", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 294, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_safe_point_gap_with_in_memory_engine\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_safe_point_gap_with_in_memory_engine\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Auto GC SafePoint Gap With TiKV", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "In Memory Engine", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 295, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 296, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) != 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "throttle-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) != 0", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 297, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_discard_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) / 10000000", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum((\n tikv_scheduler_discard_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type) / 10000000", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler discard ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 298, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Throttle duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 299, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_scheduler_throttle_cf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) != 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{cf}}", "metric": "", "query": "((\n tikv_scheduler_throttle_cf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) != 0", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler throttled CF", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 300, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_throttle_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, cf, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{cf}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_throttle_action_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, cf, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flow controller actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 301, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}_l0_flow-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}_flush_flow-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, cf) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total_l0_flow-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_l0_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total_flush_flow-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_flush_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flush/L0 flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 302, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_scheduler_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "l0-{{instance}}", "metric": "", "query": "max((\n tikv_scheduler_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_scheduler_memtable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "memtable-{{instance}}", "metric": "", "query": "max((\n tikv_scheduler_memtable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_scheduler_l0_avg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg_l0-{{instance}}", "metric": "", "query": "max((\n tikv_scheduler_l0_avg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flow controller factors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 303, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n \n)) by (cf, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_engine_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",db=\"kv\"}\n \n)) by (cf, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (cf, $additional_groupby) / 10000000", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "pending-bytes {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_scheduler_pending_compaction_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (cf, $additional_groupby) / 10000000", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction pending bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Throttle time for txn storage commands in 1 minute.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 304, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Txn command throttled duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Throttle time for non-txn related processing like analyze or dag in 1 minute.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 305, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_non_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_non_txn_command_throttle_time_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Non-txn command throttled duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Flow Control", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 306, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of commands on each stage", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 307, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{stage}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{stage}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler stage total", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different priority commands", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 308, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (priority, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{priority}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (priority, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler priority commands", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of pending commands per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 309, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_contex_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_contex_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler pending commands", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of running commands per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 310, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler running commands", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total writing bytes of commands on each stage", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 311, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler writing bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of bytes used by scheduler", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 312, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"in_use\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler memory quota", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 313, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Txn Scheduler Pool Wait Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Scheduler", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 314, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time used by each level in the yatp thread pool per second. Level 0 refers to small queries.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 315, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{level}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Time used by level", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The chance that level 0 (small) tasks are scheduled in the yatp thread pool.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 316, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Level 0 chance", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of concurrently running tasks in the yatp thread pool.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 317, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(avg_over_time(\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(avg_over_time(\n tikv_scheduler_running_commands\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Running tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 318, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Wait Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Task execution time during one schedule.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 319, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Duration of One Time Slice", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Task total execution duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 320, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_exec_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_exec_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Execute Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Task schedule number of times.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 321, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_execute_times_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_execute_times_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sched-worker.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Schedule Times", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Scheduler Worker Pool", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 322, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of commands on each stage in commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 323, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "busy-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{stage}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler stage total", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 324, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_scheduler_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler command duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time which is caused by latch wait in commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 325, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler latch wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of keys read by a commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 326, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_kv_command_key_read_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_scheduler_kv_command_key_read_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler keys read", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of keys written by a commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 327, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_kv_command_key_write_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_scheduler_kv_command_key_write_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler keys written", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of each CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 328, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of lock CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 329, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"lock\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"lock\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details [lock]", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of write CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 330, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"write\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"write\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details [write]", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of default CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 331, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"default\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=\"$command\", cf=\"default\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details [default]", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on reading when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 332, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_processing_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_scheduler_processing_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler command read duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on checking memory locks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 333, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_check_mem_lock_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_check_mem_lock_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Check memory locks duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null } ], "repeat": "command", "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Scheduler - $command", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 334, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle coprocessor read requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 335, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Request duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle coprocessor read requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 336, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 337, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 338, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (reason, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{reason}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (reason, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Request Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 339, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "KV Cursor Operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 340, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "KV Cursor Operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 341, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "delete_skipped-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total RocksDB Perf Statistics", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 342, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Response Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total bytes of memory used by coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 343, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Quota", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor Overview", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 344, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when handling coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 345, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Handle duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle coprocessor requests per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 346, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}}-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (req, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Handle duration by store", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when coprocessor requests are wait for being handled", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 347, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when coprocessor requests are wait for being handled in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 348, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}}-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Wait duration by store", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 349, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (vec_type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{vec_type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (vec_type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total DAG Requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of DAG executors", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 350, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total DAG Executors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 351, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details (Table Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 352, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details (Index Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 353, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details by CF (Table Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 354, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details by CF (Index Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on checking memory locks for coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 355, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Memory lock checking duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on checking memory locks for coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 356, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory lock checking duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor Detail", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 357, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time used by each level in the yatp thread pool per second. Level 0 refers to small queries.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 358, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{level}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Time used by level", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The chance that level 0 (small) tasks are scheduled in the yatp thread pool.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 359, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Level 0 chance", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of concurrently running tasks in the yatp thread pool.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 360, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(avg_over_time(\n tikv_unified_read_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (priority, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{priority}} {{$additional_groupby}}", "metric": "", "query": "sum(avg_over_time(\n tikv_unified_read_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (priority, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Running tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 361, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Wait Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Task execution time during one schedule.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 362, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Duration of One Time Slice", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Task total execution duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 363, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_exec_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_exec_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Execute Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Task schedule number of times.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 364, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_execute_times_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_execute_times_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Schedule Times", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Unified Read Pool", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 365, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of GC tasks processed by gc_worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 366, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_tasks_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-{{task}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_tasks_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_gc_skipped_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "skipped-{{task}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_gc_skipped_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_task_fail_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "failed-{{task}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_task_fail_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_worker_too_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "gcworker-too-busy {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_worker_too_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing GC tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 367, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_gcworker_gc_task_duration_vec_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_gcworker_gc_task_duration_vec_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC tasks duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The GC duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 368, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1,(\n sum(rate(\n tidb_tikvclient_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(1,(\n sum(rate(\n tidb_tikvclient_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiDB GC seconds", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of TiDB GC worker actions", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 369, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_tikvclient_gc_worker_actions_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tidb_tikvclient_gc_worker_actions_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiDB GC worker actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Progress of ResolveLocks, the first phase of GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 370, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_tikvclient_range_task_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=~\"resolve-locks.*\"}\n \n)) by (result, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{result}} {{$additional_groupby}}", "metric": "", "query": "max((\n tidb_tikvclient_range_task_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=~\"resolve-locks.*\"}\n \n)) by (result, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "ResolveLocks Progress", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Progress of TiKV's GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 371, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum((\n tikv_gcworker_autogc_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"}\n \n)) by (instance) / sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n \n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum((\n tikv_gcworker_autogc_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"}\n \n)) by (instance) / sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n \n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiKV Auto GC Progress", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "keys / second", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 372, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_mvcc_gc_delete_versions_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_keys/s {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_mvcc_gc_delete_versions_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "SafePoint used for TiKV's Auto GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 373, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_gcworker_autogc_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_gcworker_autogc_safe_point\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\"}\n \n)) by (instance, $additional_groupby) / (2^18)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiKV Auto GC SafePoint", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The lifetime of TiDB GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 374, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=\"tikv_gc_life_time\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": null, "metric": "", "query": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=\"tikv_gc_life_time\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "GC lifetime", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The interval of TiDB GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 375, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=\"tikv_gc_run_interval\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": null, "metric": "", "query": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",type=\"tikv_gc_run_interval\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "GC interval", "transformations": [], "transparent": false, "type": "stat" }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Keys handled in GC compaction filter", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 376, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_filtered {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_skip\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_skipped {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_skip\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_mvcc_rollback\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-rollback/mvcc-lock {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_mvcc_rollback\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_orphan_versions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_orphan-versions {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_orphan_versions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_perform\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_performed-times {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_perform\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_failure-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_met\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-deletion-met {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_met\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_handled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-deletion-handled {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_handled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_wasted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-deletion-wasted {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_wasted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC in Compaction Filter", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "GC scan write details", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 377, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}-{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC scan write details", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "GC scan default details", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 378, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}-{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC scan default details", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "GC", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 379, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 380, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"waiter_manager.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{name}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"waiter_manager.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"deadlock_detect.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{name}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"deadlock_detect.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock Manager Thread CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 381, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_task_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_lock_manager_task_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock Manager Handled tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 382, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Waiter lifetime duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 383, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(max_over_time(\n tikv_lock_manager_wait_table_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(max_over_time(\n tikv_lock_manager_wait_table_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(max_over_time(\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(max_over_time(\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock Waiting Queue", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 384, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_lock_manager_detect_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_lock_manager_detect_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Deadlock detect duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 385, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_lock_manager_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Detect error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 386, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(max_over_time(\n tikv_lock_manager_detector_leader_heartbeat\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(max_over_time(\n tikv_lock_manager_detector_leader_heartbeat\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30s]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Deadlock detector leader", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 387, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_pessimistic_lock_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_pessimistic_lock_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total pessimistic locks memory size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 388, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_pessimistic_locking\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{result}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_pessimistic_locking\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "In-memory pessimistic locking result", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of active keys and waiters.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 389, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Pessimistic lock activities", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The length includes the entering transaction itself", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 390, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_lock_wait_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_lock_wait_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Lengths of lock wait queues when transaction enqueues", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration scan in-memory pessimistic locks with read lock", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 391, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "In-memory scan lock read duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Pessimistic Locking", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 392, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of tasks handled by worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 393, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Worker handled tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current pending and running tasks of worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 394, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Worker pending tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of tasks handled by future_pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 395, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FuturePool handled tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current pending and running tasks of future_pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 396, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FuturePool pending tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Task", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 397, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of requests that TiKV sends to PD", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 398, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by requests that TiKV sends to PD", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 399, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD request duration (average)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of PD heartbeat messages", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 400, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_pd_pending_heartbeat_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-pending", "metric": "", "query": "sum((\n tikv_pd_pending_heartbeat_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD heartbeats", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of peers validated by the PD worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 401, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_pd_validate_peer_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_pd_validate_peer_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD validate peers", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of reconnection between TiKV and PD", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 402, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_pd_reconnect_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(delta(\n tikv_pd_reconnect_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD reconnection", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The forward status of PD client", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 403, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_pd_request_forwarded\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{host}}", "metric": "", "query": "((\n tikv_pd_request_forwarded\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD forward status", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of TSO requests waiting in the queue.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 404, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_pd_pending_tso_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_pd_pending_tso_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Pending TSO Requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "PD", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 405, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration that recorded by inspecting messages.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 406, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_inspect_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_inspect_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Inspected duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The slow score of stores", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 407, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_slow_score\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n tikv_raftstore_slow_score\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store Slow Score", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The changing trend of the slowness on I/O operations. 'value > 0' means the related store might have a slow trend.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 408, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_slow_trend\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_slow_trend\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Slow Trend", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The changing trend of QPS on each store. 'value < 0' means the QPS has a dropping trend.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 409, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_slow_trend_result\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_slow_trend_result\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "QPS Changing Trend", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The sampling latency of recent queries. A larger value indicates that the store is more likely to be the slowest store.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 410, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_slow_trend_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_slow_trend_l0\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "AVG Sampling Latency", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The QPS of each store.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 411, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_slow_trend_result_value\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_slow_trend_result_value\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "QPS of each store", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Slow Trend Statistics", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 412, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate of Raft snapshot messages sent", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 413, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [1m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [1m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Rate snapshot message", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of snapshots in different states", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 414, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_snapshot_traffic_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_raftstore_snapshot_traffic_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot state count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time snapshot generation/apply tasks spent waiting to be executed.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 415, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-generate", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_apply_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-apply", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_apply_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Snapshot generation/apply wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when handling snapshots", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 416, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_send_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "send {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_send_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "apply {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"generate\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "generate {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"generate\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Handle snapshot duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The snapshot size (P99.99).9999", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 417, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "size {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% Snapshot size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of KV within a snapshot in .9999", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 418, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_kv_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_kv_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% Snapshot KV count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Action stats for snapshot generating and applying", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 419, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_snapshot_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, status, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{status}} {{$additional_groupby}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_snapshot_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, status, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_clean_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, status, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "clean-region-by-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_clean_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, status, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot Actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The speed of sending or receiving snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 420, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_snapshot_limit_transport_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_snapshot_limit_transport_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_snapshot_limit_generate_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-generate", "metric": "", "query": "sum(rate(\n tikv_snapshot_limit_generate_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot transport speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of snapshots waiting to be applied", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 421, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_raftstore_snapshot_pending_applies\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_raftstore_snapshot_pending_applies\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot pending applies", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Snapshot", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 422, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of resolved ts worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 423, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"resolved_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"resolved_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Resolved TS Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of advance ts worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 424, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"advance_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"advance_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Advance ts Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of scan lock worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 425, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"inc_scan.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"inc_scan.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scan lock Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The gap between resolved ts (the maximum candidate of safe-ts) and current time.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 426, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max gap of resolved-ts", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The region that has minimal resolved ts", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 427, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Min Resolved TS Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The gap between now() and the minimal (non-zero) safe ts for followers", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 428, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_follower_safe_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_follower_safe_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max gap of follower safe-ts", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The region id of the follower that has minimal safe ts", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 429, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_follower_safe_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_follower_safe_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Min Safe TS Follower Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The gap between resolved ts of leaders and current time", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 430, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max gap of resolved-ts in region leaders", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The region that its leader has minimal resolved ts.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 431, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Min Leader Resolved TS Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when handle a check leader request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 432, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_resolved_ts_check_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_resolved_ts_check_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Check leader duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Bucketed histogram of region count in a check leader request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 433, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% CheckLeader request region count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Bucketed histogram of the check leader request size", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 434, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-check-num", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% CheckLeader request size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of fail to advance resolved-ts", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 435, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_resolved_ts_fail_advance_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{reason}}", "metric": "", "query": "sum(delta(\n tikv_resolved_ts_fail_advance_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_check_stale_peer\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-stale-peer", "metric": "", "query": "sum(delta(\n tikv_raftstore_check_stale_peer\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Fail advance ts count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total bytes in memory of resolved-ts observe regions's lock heap", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 436, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_resolved_ts_lock_heap_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "avg((\n tikv_resolved_ts_lock_heap_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock heap size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The backoff duration before starting initial scan", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 437, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Initial scan backoff duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The status of resolved-ts observe regions", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 438, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_region_resolve_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_resolved_ts_region_resolve_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Observe region status", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total bytes of pending commands in the channel", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 439, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_resolved_ts_channel_penging_cmd_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "avg((\n tikv_resolved_ts_channel_penging_cmd_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Pending command size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Resolved TS", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 440, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 441, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 442, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"queue|exec_download\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "(DL){{type}}-99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"queue|exec_download\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "(AP){{type}}-99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "P99 RPC Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 443, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (instance, request) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{request}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (instance, request) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-{{request}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Ops", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 444, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 445, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_apply_cache_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_apply_cache_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cache Events", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 446, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Overall RPC Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 447, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"exec_download\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"exec_download\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Read File into Memory Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 448, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"queuing\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"queuing\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Queuing Time", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 449, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_apply_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_apply_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply Request Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 450, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_download_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_download_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Downloaded File Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 451, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_apply_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_apply_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply Batch Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 452, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"get_permit\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"get_permit\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Blocked by Concurrency Time", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 453, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply Request Speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 454, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_import_apply_cached_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_import_apply_cached_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cached File in Memory", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 455, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type!=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Engine Requests Unfinished", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 456, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply Time", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 457, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"raftstore-.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Store Memory Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Point In Time Restore", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 458, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 459, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"b.*k.*w.*k.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "backup-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"b.*k.*w.*k.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_io\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "backup-io-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_io\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_backup_softlimit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "backup-auto-throttle-{{instance}}", "metric": "", "query": "((\n tikv_backup_softlimit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup CPU Utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 460, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_backup_thread_pool_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_backup_thread_pool_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup Thread Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 461, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_backup_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, error) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{error}}", "metric": "", "query": "sum(delta(\n tikv_backup_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, error) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 462, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Write CF SST Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 463, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",cf=\"default\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Default CF SST Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 464, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total", "metric": "", "query": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{cf}}", "metric": "", "query": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, cf) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup SST Generation Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 465, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Scan SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 466, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"scan\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Scan SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 467, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"save.*\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"save.*\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Save SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 468, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-99.9% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum((\n tikv_backup_range_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) / sum((\n tikv_backup_range_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-avg {{$additional_groupby}}", "metric": "", "query": "(sum((\n tikv_backup_range_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) / sum((\n tikv_backup_range_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup SST Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 469, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "External Storage Create Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 470, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_external_storage_create_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_external_storage_create_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "External Storage Create Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 471, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Checksum Request Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 472, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, device) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{device}}", "metric": "", "query": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, device) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IO Utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 473, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "import-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, tid) > 0", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "import-{{instance}}-{{tid}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, tid) > 0", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "import-count-{{instance}}", "metric": "", "query": "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import CPU Utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 474, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Thread Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 475, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_import_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, error, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{error}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_import_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, error, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 476, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{request}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{request}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_import_rpc_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) / sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{request}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_import_rpc_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) / sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{request}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 477, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{request}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Ops", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 478, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 479, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=~\"download|write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=~\"download|write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Write/Download RPC Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 480, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"queue\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"queue\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Wait Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 481, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"read\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"read\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Read SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 482, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"rewrite\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"rewrite\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Rewrite SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 56 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 483, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",request=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Ingest RPC Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 56 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 484, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_ingest_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_ingest_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Ingest SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 56 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 485, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_ingest_byte_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_ingest_byte_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Ingest SST Bytes", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 486, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total", "metric": "", "query": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Download SST Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 487, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_import_local_write_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_import_local_write_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Local Write keys", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 488, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_local_write_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_local_write_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Local Write bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 489, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "sum", "metric": "", "query": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL Expired", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 490, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_cloud_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cloud, req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cloud}}-{{req}}", "metric": "", "query": "sum(rate(\n tikv_cloud_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (cloud, req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "cloud request", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Backup & Import", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 491, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": [ { "options": { "0": { "color": "red", "index": null, "text": "Disabled" }, "1": { "color": "green", "index": null, "text": "Enabled" } }, "type": "value" } ], "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 492, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_log_backup_enabled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "((\n tikv_log_backup_enabled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Endpoint Status", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": [ { "options": { "0": { "color": "green", "index": null, "text": "Running" }, "1": { "color": "yellow", "index": null, "text": "Paused" }, "2": { "color": "red", "index": null, "text": "Error" } }, "type": "value" } ], "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 493, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "min((\n tikv_log_backup_task_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "min((\n tikv_log_backup_task_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Task Status", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 494, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "name" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "tidb_log_backup_advancer_owner > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "tidb_log_backup_advancer_owner > 0", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Advancer Owner", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average flush size of last 30mins.", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 495, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(increase(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) / sum(increase(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [30m]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "(sum(increase(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) / sum(increase(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [30m]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Average Flush Size", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The current total flushed file number of this run.", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": 0, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 496, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) > 0", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flushed Files (Last 30m) Per Host", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": 0, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 497, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [30m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [30m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flush Times (Last 30m)", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "This is the summary of the size has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 498, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Total Flushed Size (Last 30m)", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": 0, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 499, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [30m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flush Files (Last 30m)", "transformations": [], "transparent": false, "type": "stat" }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of log backup threads. \n**(Note this is the average usage for a period of time, some peak of CPU usage may be lost.)**", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 500, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",name=~\"backup_stream|log-backup-scan(-[0-9]+)?\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 501, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_handle_kv_batch_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_handle_kv_batch_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Handle Event Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The data rate of initial scanning emitting events.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 502, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_incremental_scan_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_incremental_scan_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scan Generate Event Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 503, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(time() * 1000 - max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (task) / 262144 > 0)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ task }}", "metric": "", "query": "(time() * 1000 - max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (task) / 262144 > 0)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Abnormal Checkpoint TS Lag", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The estimated memory usage by the streaming backup module.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 504, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_log_backup_heap_memory\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_log_backup_heap_memory\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Of Events", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 505, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-total", "metric": "", "query": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Observed Region Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The errors met when backing up.\n**They are retryable, don't worry.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 506, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The errors met when backing up.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 507, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_fatal_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_fatal_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [1m]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Fatal Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 508, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Current Time", "bars": false, "dashes": true, "fill": 0, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (task) / 262144 > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{task}}", "metric": "", "query": "max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (task) / 262144 > 0", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "time() * 1000", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Current Time", "metric": "", "query": "time() * 1000", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Checkpoint TS of Tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "dateTimeAsIsoNoDateIfToday", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of flushing a batch of file.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 509, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_flush_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_flush_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_files\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flush Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of scanning the initial data from local DB and transform them into apply events.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 510, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_initial_scan_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_initial_scan_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Initial scanning duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of converting a raft request into a apply event.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 511, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"to_stream_event\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"to_stream_event\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Convert Raft Event duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of waiting the mutex of the controller.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 512, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"get_router_lock\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"get_router_lock\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Wait for Lock Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The number of KV-modify of each raft command observed.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 513, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_handle_kv_batch_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_handle_kv_batch_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Command Batch Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The total cost of saving an event into temporary file.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 514, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_to_temp_file\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=~\"save_to_temp_file\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Save to Temp File Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The total cost of writing a event into temporary file.\nComparing to the ***Save*** duration, it doesn't contain the time cost of routing the task by range / task.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 515, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=\"write_to_tempfile\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=\"write_to_tempfile\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Write to Temp File Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of collecting metadata and call the UNIX system call *write* for each event.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 516, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=\"syscall_write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",stage=\"syscall_write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "System Write Call Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal message type count.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 517, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{message}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Internal Message Type", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 518, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{message}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Internal Message Handling Duration (P99)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 519, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{message}}", "metric": "", "query": "histogram_quantile(0.9,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Internal Message Handling Duration (P90)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal read throughput of RocksDB during initial scanning. This panel can roughly present the read through to the hard disk of initial scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 520, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op=~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scan RocksDB Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Misc statistics of RocksDB during initial scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 521, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op!~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf, op) > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{op}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",op!~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf, op) > 0", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scan RocksDB Operation", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The reason of triggering initial scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 522, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_initial_scan_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{reason}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_initial_scan_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scanning Trigger Reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The task status of initial scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 523, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_log_backup_pending_initial_scan\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (stage) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{stage}}", "metric": "", "query": "sum((\n tikv_log_backup_pending_initial_scan\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (stage) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scanning Task Status", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 524, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_metadata_key_operation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_metadata_key_operation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Checkpoint Key Putting", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 525, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tidb_log_backup_advancer_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"checkpoint\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tidb_log_backup_advancer_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"checkpoint\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Request Checkpoint Batch Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 526, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"tick\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"tick\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Tick Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The reason of advancer failed to be advanced.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 527, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_region_request_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason!=\"retryable-scan-region\"}\n [$__rate_interval]\n)) by (reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{reason}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_region_request_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",reason!=\"retryable-scan-region\"}\n [$__rate_interval]\n)) by (reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Checkpoint Failure Reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The result of getting region checkpoints.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 528, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_region_request\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (result) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{result}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_region_request\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (result) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request Result", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 529, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ step }}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Tick Duration (P99)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 530, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ step }}", "metric": "", "query": "histogram_quantile(0.9,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Tick Duration (P90)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The frequent of getting region level checkpoint.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 531, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"get-regions-in-range\"}\n [$__rate_interval]\n)) by (step, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{step}}-{{instance}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"get-regions-in-range\"}\n [$__rate_interval]\n)) by (step, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Get Region Operation Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The variant of checkpoint group.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 532, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"try-advance\"}\n [$__rate_interval]\n)) by (step, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{step}}-{{instance}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",step=\"try-advance\"}\n [$__rate_interval]\n)) by (step, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Try Advance Trigger Time", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Backup Log", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 533, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 534, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, state) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{state}}", "metric": "", "query": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, state) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-total", "metric": "", "query": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads state", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 535, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n sum(rate(\n tikv_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, io) > 1024\n \n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "topk(20,(\n sum(rate(\n tikv_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name, io) > 1024\n \n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads IO", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 536, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n max(rate(\n tikv_thread_voluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "topk(20,(\n max(rate(\n tikv_thread_voluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Thread Voluntary Context Switches", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 537, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n max(rate(\n tikv_thread_nonvoluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "topk(20,(\n max(rate(\n tikv_thread_nonvoluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Thread Nonvoluntary Context Switches", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Threads", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 538, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 539, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_allocator_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n tikv_allocator_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Allocator Stats", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 540, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) - sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{thread_name}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) - sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Send Allocated(+) / Release Received(-) Bytes Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 541, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{thread_name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Allocated Bytes Rate per Thread", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 542, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{thread_name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Released Bytes Rate per Thread", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 543, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_allocator_thread_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"mapped\"}\n \n)) by (thread_name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{thread_name}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_allocator_thread_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"mapped\"}\n \n)) by (thread_name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Mapped Allocation per Thread", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 544, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_allocator_arena_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_allocator_arena_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Arena Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Memory", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 545, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 546, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_resource_control_background_task_wait_duration\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{resource_group}}", "metric": "", "query": "sum(rate(\n tikv_resource_control_background_task_wait_duration\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Background Task Total Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memory usage of the resource control module.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 547, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resource_control_priority_quota_limit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, priority) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{priority}}", "metric": "", "query": "sum((\n tikv_resource_control_priority_quota_limit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance, priority) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Priority Quota Limit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Resource Control", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 548, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The 99 quantile durtion of status server API requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 549, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{path}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{path}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_status_server_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) / sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{path}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_status_server_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) / sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{path}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Status API Request Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 550, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{path}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Status API Request (op/s)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Status Server", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 551, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total number of encryption data keys in use", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 552, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_encryption_data_key_storage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_encryption_data_key_storage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encryption data keys", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of files being encrypted", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 553, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_encryption_file_num\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_encryption_file_num\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encrypted files", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Flag to indicate if encryption is initialized", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 554, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_encryption_is_initialized\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_encryption_is_initialized\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encryption initialized", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total size of encryption meta files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 555, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_encryption_meta_file_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{instance}}", "metric": "", "query": "((\n tikv_encryption_meta_file_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encryption meta files size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 556, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"encrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "encrypt-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"encrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"decrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "decrypt-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",metric=\"decrypt_data_nanos\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encrypt/decrypt data nanos", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Writing or reading file duration (second)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 557, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_encryption_write_read_file_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_encryption_write_read_file_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_encryption_write_read_file_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_encryption_write_read_file_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read/write encryption meta duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Encryption", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 558, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 559, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_expire_kv_count_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_ttl_expire_kv_count_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL expire count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 560, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_expire_kv_size_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_ttl_expire_kv_size_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL expire size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 561, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_ttl_checker_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_ttl_checker_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"region\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL check progress", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 562, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_checker_actions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_ttl_checker_actions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL checker actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing GC tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "fill": 0, "fillGradient": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 563, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "count", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "avg", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_ttl_checker_compact_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_ttl_checker_compact_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL checker compact duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 564, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_ttl_checker_poll_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"tikv_gc_run_interval\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": null, "metric": "", "query": "max((\n tikv_ttl_checker_poll_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",type=\"tikv_gc_run_interval\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "TTL checker poll interval", "transformations": [], "transparent": false, "type": "stat" } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "TTL", "transformations": [], "transparent": false, "type": "row" } ], "refresh": "1m", "rows": [], "schemaVersion": 14, "sharedCrosshair": false, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "k8s_cluster", "multi": false, "name": "k8s_cluster", "options": [], "query": "label_values(tikv_engine_block_cache_size_bytes, k8s_cluster)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", "options": [], "query": "label_values(tikv_engine_block_cache_size_bytes{k8s_cluster =\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "db", "multi": true, "name": "db", "options": [], "query": "label_values(tikv_engine_block_cache_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "command", "multi": true, "name": "command", "options": [], "query": "query_result(tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"} != 0)", "refresh": 2, "regex": "/\\btype=\"([^\"]+)\"/", "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": ".*", "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "instance", "multi": true, "name": "instance", "options": [], "query": "label_values(tikv_engine_size_bytes{k8s_cluster =\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": true, "label": "titan_db", "multi": true, "name": "titan_db", "options": [], "query": "label_values(tikv_engine_titandb_num_live_blob_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": {}, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": false, "label": "additional_groupby", "multi": false, "name": "additional_groupby", "options": [ { "selected": false, "text": "none", "value": "none" }, { "selected": false, "text": "instance", "value": "instance" } ], "query": "none,instance", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "custom", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "hidden": false, "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Test-Cluster-TiKV-Details", "uid": "RDVQiEzZz", "version": 0 }