{ "__inputs": [ { "description": "", "label": "test-cluster", "name": "DS_TEST-CLUSTER", "pluginId": "prometheus", "pluginName": "Prometheus", "type": "datasource" } ], "annotations": { "list": [] }, "description": "", "editable": true, "gnetId": null, "graphTooltip": 1, "hideControls": false, "id": null, "links": [], "panels": [ { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 1, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Write Pipeline Composition", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 2, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Write Raft Log-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_append_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Propose Wait-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Apply Wait-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Replicate Raft Log-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Apply Duration-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Pipeline Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Read Duration Composition", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 3, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Get Snapshot-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Cop Wait-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Cop Handle-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cop Read Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Duration", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 4, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Approximate durable local used size per TiKV instance (row SST catalog + raft).", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 5, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all_used\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all_used\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The available capacity size of each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 6, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"available\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"available\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Available size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The capacity size per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 7, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"capacity\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"capacity\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Capacity size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU usage of each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 8, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "quota-{{instance}}", "metric": "", "query": "((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max-quota", "metric": "", "query": "max((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "min((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "min-quota", "metric": "", "query": "min((\n tikv_server_cpu_cores_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memory usage per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 9, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "quota-{{instance}}", "metric": "", "query": "((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max-quota", "metric": "", "query": "max((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "min((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "min-quota", "metric": "", "query": "min((\n tikv_server_memory_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The I/O utilization per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 10, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{device}}", "metric": "", "query": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IO utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total bytes of read and write in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 11, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-write", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"wal_file_bytes\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-read", "metric": "", "query": "sum(rate(\n tikv_engine_flow_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_engine_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-in-memory-engine-read", "metric": "", "query": "sum(rate(\n tikv_in_memory_engine_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"bytes_read|iter_bytes_read\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "MBps", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of leaders on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 12, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of the gRPC message failures", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 13, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-grpc-msg-fail", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"noop\"}\n [$__rate_interval]\n)) by (instance) < 1", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-pd-heartbeat", "metric": "", "query": "sum(delta(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"noop\"}\n [$__rate_interval]\n)) by (instance) < 1", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Errps", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of leaders on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 14, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"leader\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"leader\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Leader", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Regions and Buckets on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 15, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"buckets\"}\n \n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-buckets", "metric": "", "query": "sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"buckets\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Physical sizes summed over region-leader shards (unique when summed across stores). ", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "fillGradient": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 16, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"row_size\"}\n \n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-row", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"row_size\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"leader_.*_size\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"leader_.*_size\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Leader data size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "TiKV uptime since the last restart", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 17, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(time() - ((\n process_start_time_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(time() - ((\n process_start_time_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n \n)) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Uptime", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Cluster", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 18, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.0, "yaxis": "left" } ] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 19, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_critical_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.0, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Critical error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "\nShows ServerIsBusy responses constructed for clients and subsystem pressure signals\nthat can make a TiKV instance temporarily unavailable.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 20, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_is_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "server-is-busy-{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_server_is_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "scheduler-{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_unified_read_pool_rejected_tasks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, name, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read-pool-{{instance}}-{{name}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_unified_read_pool_rejected_tasks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, name, reason) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "channelfull-{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",reason=~\"deadline_exceeded|max_pending_tasks_exceeded|cpu_throttled|memory_quota_exceeded|server_is_busy\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "coprocessor-{{instance}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",reason=~\"deadline_exceeded|max_pending_tasks_exceeded|cpu_throttled|memory_quota_exceeded|server_is_busy\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write_stall_percentile99\",db=~\"$db\"}\n \n)) by (instance, db) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "stall-{{instance}}-{{db}}", "metric": "", "query": "avg((\n tikv_engine_write_stall\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write_stall_percentile99\",db=~\"$db\"}\n \n)) by (instance, db) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "store-write-channelfull-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_msg_block_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_process_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n tikv_raftstore_process_busy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Server is busy", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of reporting failure messages", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 21, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_report_failure_msg_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance, store_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}-to-{{store_id}}", "metric": "", "query": "sum(rate(\n tikv_server_report_failure_msg_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance, store_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Server report failures", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of different raftstore errors on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 22, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status!~\"success|all\"}\n [$__rate_interval]\n)) by (instance, status) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{status}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status!~\"success|all\"}\n [$__rate_interval]\n)) by (instance, status) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raftstore error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of scheduler errors per type on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 23, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"snapshot_err|prepare_write_err\"}\n [$__rate_interval]\n)) by (instance, stage) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{stage}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"snapshot_err|prepare_write_err\"}\n [$__rate_interval]\n)) by (instance, stage) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of different coprocessor errors on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 24, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of gRPC message errors per type on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 25, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC message error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of dropped leaders per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 26, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"leader\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"leader\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Leader drop", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of missing leaders per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 27, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_leader_missing\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_leader_missing\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Leader missing", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of advertised address probe failures per TiKV instance, grouped by loopback, unresolved, and timeout reasons", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 28, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_server_advertise_addr_probe_failure_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{endpoint}}-{{reason}}", "metric": "", "query": "((\n tikv_server_advertise_addr_probe_failure_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Advertised Address Probe Failures", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of Log Replication Reject caused by follower memory insufficient", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 29, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_append_rejects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_append_rejects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Log Replication Rejected", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Errors", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 30, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The size of each column family", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 31, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_engine_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_engine_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CF size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of channel full errors on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 32, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_channel_full_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Channel full", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of leaders being written on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 33, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Active written leaders", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 34, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Approximate region size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The approximate Region size", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 35, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_region_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_region_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_region_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_region_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Approximate region size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Histogram of huge mem table bytes for regions", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 36, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_region_huge_mem_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_region_huge_mem_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_region_huge_mem_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_region_huge_mem_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_region_huge_mem_table_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_region_huge_mem_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_region_huge_mem_table_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_region_huge_mem_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_region_huge_mem_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_region_huge_mem_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Huge memtable size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Histogram of huge l0 table bytes for regions", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 37, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_region_huge_l0_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_region_huge_l0_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_region_huge_l0_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_region_huge_l0_table_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_region_huge_l0_table_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_region_huge_l0_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_region_huge_l0_table_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_region_huge_l0_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_region_huge_l0_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_region_huge_l0_table_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Huge l0 table size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 38, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_region_written_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_region_written_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region written bytes", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average rate of writing bytes to Regions per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 39, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_region_written_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_region_written_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_bytes_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region average written bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 40, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_region_written_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_region_written_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region written keys", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average rate of written keys to Regions per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 41, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_region_written_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_region_written_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_region_written_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region average written keys", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Bucketed histogram of clear overlap region duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 42, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_clear_overlap_region_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Clear overlap region duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The key size for every put of apply worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 43, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_apply_key_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_apply_key_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Written key size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 44, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_apply_value_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_apply_value_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Written value size for every put of apply worker", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of peers in hibernated state", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 45, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_hibernated_peer_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, state) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{state}}", "metric": "", "query": "sum((\n tikv_raftstore_hibernated_peer_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, state) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Hibernate Peers", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 46, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raftstore-.*\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{name}}", "metric": "", "query": "((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raftstore-.*\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n raft_engine_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-raft-engine", "metric": "", "query": "((\n raft_engine_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory trace", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 47, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raft_entries_evict_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raft_entries_evict_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Entry Cache Evicts", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 48, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_address_resolve_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_address_resolve_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Resolve address duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 49, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, priority, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{priority}}-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_yatp_pool_schedule_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, priority, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Thread Pool Schedule Wait Duration$optional_quantile", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average schedule wait duration for thread pools", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 50, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{priority}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, priority, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "sched-tokio-pool-mean-{{instance}}", "metric": "", "query": "((\n tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "unified-read-pool-tokio-mean-{{instance}}", "metric": "", "query": "((\n tikv_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 1.0, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Average Thread Pool Schedule Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 51, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "copr-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_time\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk IO time per second", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ns", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 52, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "copr-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"block_read_byte\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk IO bytes per second", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ns", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Server", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 53, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different kinds of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 54, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{priority}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC message count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different kinds of gRPC message which is failed", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 55, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_msg_fail_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC message failed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The $optional_quantile of execution time of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 56, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{priority}}-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_grpc_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"kv_gc\"}\n [$__rate_interval]\n)) by (type, priority, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC message duration$optional_quantile", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average execution time of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 57, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{priority}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_grpc_msg_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) / sum(rate(\n tikv_grpc_msg_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, priority, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Average gRPC message duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The 99.99% wait time of gRPC batch commands", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 58, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_grpc_batch_commands_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "P9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_grpc_batch_commands_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC batch commands wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The $optional_quantile of execution time of gRPC message", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 59, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_grpc_req_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "request-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_grpc_req_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_grpc_resp_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "response-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_grpc_resp_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_grpc_req_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_req_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg request {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_server_grpc_req_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_req_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_grpc_resp_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_resp_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg response {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_server_grpc_resp_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_grpc_resp_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_request_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "kv get batch-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_request_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_request_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_request_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg kv batch {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_server_request_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_request_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC batch size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 60, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_raft_message_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_raft_message_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_raft_message_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_raft_message_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_server_raft_message_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_server_raft_message_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "raft message batch size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The QPS of different sources of gRPC request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 61, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_request_source_counter_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{source}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_request_source_counter_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC request sources QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": true, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of different sources of gRPC request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 62, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": false, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_request_source_duration_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{source}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_request_source_duration_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (source, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC request sources duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The QPS of different resource groups of gRPC request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 63, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_grpc_resource_group_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_grpc_resource_group_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC resource group QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": null, "mappings": [ { "options": { "0": { "color": "green", "index": null, "text": "none" }, "1": { "color": "yellow", "index": null, "text": "deflate" }, "2": { "color": "red", "index": null, "text": "gzip" } }, "type": "value" } ], "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 64, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_grpc_compression_type_for_msg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "((\n tikv_grpc_compression_type_for_msg\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "gRPC Compression Status for Msg", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": null, "mappings": [ { "options": { "0": { "color": "green", "index": null, "text": "none" }, "1": { "color": "yellow", "index": null, "text": "deflate" }, "2": { "color": "red", "index": null, "text": "gzip" } }, "type": "value" } ], "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 65, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_grpc_compression_type_for_raft\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "((\n tikv_grpc_compression_type_for_raft\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "gRPC Compression Status for Raft Replication", "transformations": [], "transparent": false, "type": "stat" } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "gRPC", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 66, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total count of different kinds of commands received", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 67, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_command_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_command_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage command total", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of engine asynchronous request errors", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 68, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status!~\"all|success\"}\n [$__rate_interval]\n)) by (status, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{status}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status!~\"all|success\"}\n [$__rate_interval]\n)) by (status, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async request error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by processing asynchronous write requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 69, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Storage async write duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async write duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 70, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by processing asynchronous snapshot requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 71, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async snapshot duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 72, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async snapshot duration without the involving of raftstore", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 73, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration (pure local read)", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage async snapshot duration without the involving of raftstore", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 74, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_local_read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async snapshot duration (pure local read)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Read index propose wait duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 75, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Read index propose wait duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Read index propose wait duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 76, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_propose_wait\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read index propose wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Read index confirm duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 77, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Read index confirm duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Read index confirm duration associated with async snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 78, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot_read_index_confirm\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read index confirm duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The max_ts in the concurrency manager", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 79, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "tikv_concurrency_manager_max_ts_limit", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max_ts_limit-{{instance}}", "metric": "", "query": "tikv_concurrency_manager_max_ts_limit", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "tikv_concurrency_manager_max_ts", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max_ts-{{instance}}", "metric": "", "query": "tikv_concurrency_manager_max_ts", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Concurrency manager max-ts", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Storage", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 80, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 81, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/.*-total/", "bars": false, "fill": 1, "fillBelowTo": null, "lines": true, "yaxis": 2, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_local_read_reject_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-reject-by-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_local_read_reject_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_local_read_executed_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-total", "metric": "", "query": "sum(rate(\n tikv_raftstore_local_read_executed_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_local_read_executed_stale_read_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-stale-read", "metric": "", "query": "sum(rate(\n tikv_raftstore_local_read_executed_stale_read_requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Local reader requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Local Reader", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 82, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of raftstore thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 83, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(raftstore|rs|rfstore)_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(raftstore|rs|rfstore)_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft store CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of raftstore thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 84, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(raftstore|rs|rfstore)_.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(raftstore|rs|rfstore)_.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft store CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of async apply", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 85, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"apply_[0-9]+|apply_follower_[0-9]+\"}\n [$__rate_interval]\n)) by (instance, name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{name}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"apply_[0-9]+|apply_follower_[0-9]+\"}\n [$__rate_interval]\n)) by (instance, name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Async apply CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of async apply", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 86, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"apply_[0-9]+|apply_follower_[0-9]+\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{name}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"apply_[0-9]+|apply_follower_[0-9]+\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Async apply CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of raft log writer thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.8, "yaxis": "left" } ] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 87, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raft_io.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raft_io.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.8, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Raft async writer CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of raft log writer thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.8, "yaxis": "left" } ] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 88, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raft_io.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raft_io.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.8, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Raft async writer CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of the scheduler pools", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 89, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched_tokio_pool.*|sched_yatp_pool.*|sched_background.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched_tokio_pool.*|sched_yatp_pool.*|sched_background.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler pool CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of the scheduler pools", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 90, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched_tokio_pool.*|sched_yatp_pool.*|sched_background.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched_tokio_pool.*|sched_yatp_pool.*|sched_background.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler pool CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of gRPC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 91, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"grpc.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"grpc.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC poll CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of gRPC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 92, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"grpc.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"grpc.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "gRPC poll CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 93, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"txn_chunk_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"txn_chunk_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "KvEngine worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 94, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"txn_chunk_worker.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"txn_chunk_worker.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "KvEngine worker CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 95, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sub_compact_wkr.*|compact_wal_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sub_compact_wkr.*|compact_wal_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "RfEngine worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 96, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sub_compact_wkr.*|compact_wal_worker.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sub_compact_wkr.*|compact_wal_worker.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "RfEngine worker CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of the unified read pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 97, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified_read_po.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified_read_po.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Unified read pool CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of the unified read pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 98, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified_read_po.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified_read_po.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 7.2, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Unified read pool CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 99, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"background.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"background.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Background worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 100, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"background.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"background.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Background worker CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 101, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"gc_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"gc_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 102, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"gc_worker.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"gc_worker.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC worker CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 103, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 104, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 105, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(backup_worker|bkwkr|backup_endpoint).*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(backup_worker|bkwkr|backup_endpoint).*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 106, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(backup_worker|bkwkr|backup_endpoint).*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(backup_worker|bkwkr|backup_endpoint).*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 107, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(cdcwkr.*|tso|cdc_.*)\"}\n [$__rate_interval]\n)) by (instance, name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{name}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(cdcwkr.*|tso|cdc_.*)\"}\n [$__rate_interval]\n)) by (instance, name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CDC worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 108, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(cdcwkr.*|tso|cdc_.*)\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{name}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(cdcwkr.*|tso|cdc_.*)\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CDC worker CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of raftstore thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 109, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"tso_worker\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"tso_worker\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TSO worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of raftstore thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 110, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"tso_worker\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"tso_worker\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TSO worker CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of IME threads", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 111, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{name}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance, name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IME CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of IME threads", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 112, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{name}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"ime.*\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IME CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of IA threads", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 105 }, "height": null, "hideTimeOverride": false, "id": 113, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"s3fifo.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"s3fifo.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of IA threads", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 105 }, "height": null, "hideTimeOverride": false, "id": 114, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"s3fifo.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"s3fifo.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of DFS worker threads", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 112 }, "height": null, "hideTimeOverride": false, "id": 115, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(builtin_dfs.*|memory_fs.*|s3_client.*)\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(builtin_dfs.*|memory_fs.*|s3_client.*)\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "DFS worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of DFS worker threads", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 112 }, "height": null, "hideTimeOverride": false, "id": 116, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(builtin_dfs.*|memory_fs.*|s3_client.*)\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(builtin_dfs.*|memory_fs.*|s3_client.*)\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "DFS worker CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of Cloud worker threads", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 119 }, "height": null, "hideTimeOverride": false, "id": 117, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"worker_server.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"worker_server.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cloud worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of Cloud worker threads", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 119 }, "height": null, "hideTimeOverride": false, "id": 118, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"worker_server.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"worker_server.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cloud worker CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 126 }, "height": null, "hideTimeOverride": false, "id": 119, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"txn_chunk_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"txn_chunk_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "KvEngine worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 126 }, "height": null, "hideTimeOverride": false, "id": 120, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name!~\"rocksdb.*\"}\n [$__rate_interval]\n)) by (instance, name) > 0.8\n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{instance}}", "metric": "", "query": "topk(20,(\n sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name!~\"rocksdb.*\"}\n [$__rate_interval]\n)) by (instance, name) > 0.8\n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Busy Threads (>80%)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Thread CPU", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 121, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "PSI averages (percent) for some/full stalls over 10s/60s windows.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 122, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"cpu\",level=\"some\",window=\"avg10\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "some-avg10-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"cpu\",level=\"some\",window=\"avg10\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"cpu\",level=\"some\",window=\"avg60\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "some-avg60-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"cpu\",level=\"some\",window=\"avg60\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"cpu\",level=\"full\",window=\"avg10\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "full-avg10-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"cpu\",level=\"full\",window=\"avg10\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"cpu\",level=\"full\",window=\"avg60\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "full-avg60-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"cpu\",level=\"full\",window=\"avg60\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU PSI", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "PSI averages (percent) for some/full stalls over 10s/60s windows.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 123, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"io\",level=\"some\",window=\"avg10\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "some-avg10-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"io\",level=\"some\",window=\"avg10\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"io\",level=\"some\",window=\"avg60\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "some-avg60-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"io\",level=\"some\",window=\"avg60\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"io\",level=\"full\",window=\"avg10\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "full-avg10-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"io\",level=\"full\",window=\"avg10\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"io\",level=\"full\",window=\"avg60\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "full-avg60-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"io\",level=\"full\",window=\"avg60\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IO PSI", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "PSI averages (percent) for some/full stalls over 10s/60s windows.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 124, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"memory\",level=\"some\",window=\"avg10\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "some-avg10-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"memory\",level=\"some\",window=\"avg10\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"memory\",level=\"some\",window=\"avg60\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "some-avg60-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"memory\",level=\"some\",window=\"avg60\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"memory\",level=\"full\",window=\"avg10\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "full-avg10-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"memory\",level=\"full\",window=\"avg10\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"memory\",level=\"full\",window=\"avg60\"}\n \n)) / 100)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "full-avg60-{{instance}}", "metric": "", "query": "(((\n tikv_pressure_stall_average\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",resource=\"memory\",level=\"full\",window=\"avg60\"}\n \n)) / 100)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory PSI", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Pressure Stall (PSI)", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 125, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The throughput of disk write per IO type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 126, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"write\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"write\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write IO bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The throughput of disk read per IO type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 127, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"read\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"read\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_io_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=\"read\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read IO bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The threshold of disk IOs per priority", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 128, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_rate_limiter_max_bytes_per_sec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "avg((\n tikv_rate_limiter_max_bytes_per_sec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IO threshold", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "IO rate limiter request wait duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 129, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_rate_limiter_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Rate Limiter Request Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "IO Breakdown", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 130, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by processing asynchronous write requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 131, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_engine_async_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_engine_async_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_engine_async_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"write\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage async write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The store time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 132, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The apply time duration of each request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 133, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_apply_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_apply_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_apply_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time from request scheduling to when it is handled by Raftstore", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 134, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store propose wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time from request scheduling to when a batch of requests is formed and prepared to be proposed to Raft", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 135, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_batch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store batch wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time from request scheduling to just before it is sent to the store writer thread", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 136, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_send_to_queue_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store send to write queue duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time from request scheduling to just before it is sent to followers", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 137, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_send_proposal_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store send proposal duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time from request scheduling to just before it is written to Raft Engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 138, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_before_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store before write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time from request scheduling to when it is written to Raft Engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 139, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_write_end_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store write end duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time from request scheduling to when its associated ready is persisted on the leader", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 140, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store persist duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time from request scheduling to when it is committed; at the time of commit, it has not been persisted on the leader", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 141, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_commit_not_persist_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store commit but not persist duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time from request scheduling to when it is committed; at the time of commit, it has been persisted on the leader", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 142, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_wf_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store commit and persist duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Waterfall", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 143, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time taken by Raftstore to complete processing a poll round, which includes a batch of region peers", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 144, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Process ready duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time taken by Raftstore to complete processing a poll round, which includes a batch of region peers", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 145, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_raft_process_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_raft_process_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_raft_process_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Process ready duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration taken by sending raft logs to channels of followers.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 146, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_store_write_send_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_store_write_send_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write send duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration taken by sending raft logs to channels of followers.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 147, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_send_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Store write send duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft appends log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 148, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Append log duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft appends log on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 149, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_raftdb_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Append log duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft commits log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 150, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Commit log duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft commits log on each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 151, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_commit_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_commit_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_commit_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Commit log duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when Raft applies log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 152, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply log duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed for Raft to apply logs per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 153, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_log_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_apply_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_apply_log_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_apply_log_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Apply log duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft IO", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 154, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The proposal count of a Regions in a tick", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 155, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_proposal_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_proposal_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft proposals per ready", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of proposals per type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 156, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"local_read|normal|read_index\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"local_read|normal|read_index\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft read/write proposals", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of read proposals which are made by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 157, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"local_read|read_index\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"local_read|read_index\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft read proposals per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of write proposals which are made by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 158, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"normal\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"normal\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft write proposals per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of write apply", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 159, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_write_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_write_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft write apply", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of change_set apply", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 160, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kvengine_apply_change_set_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n kvengine_apply_change_set_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft change_set apply", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of change set proposals which are made by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 161, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"change_set\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"change_set\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft change set proposals per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of change set apply which are made by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 162, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_write_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"change_set|shard_ingest\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_write_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"change_set|shard_ingest\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft change set apply per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each proposal", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 163, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Propose wait duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each proposal in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 164, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_request_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Propose wait duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each store write task", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 165, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write wait duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each store write task in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 166, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_store_write_task_wait_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Store write wait duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each apply task", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 167, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply wait duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The wait time of each apply task in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 168, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_apply_wait_time_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Apply wait duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The handle duration of each store write task msg", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 169, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_store_write_handle_msg_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_store_write_handle_msg_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write handle msg duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The distribution of write trigger size", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 170, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_store_write_trigger_wb_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_store_write_trigger_wb_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Store write trigger size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate at which peers propose logs", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 171, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_propose_log_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_propose_log_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft propose speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate at which peers propose logs", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 172, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_store_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "store-{{type}}-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_store_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "apply-{{type}}-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_apply_perf_context_time_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Perf Context duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Propose", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 173, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different ready type of Raft", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 174, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_ready_handled_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_ready_handled_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_process_duration_secs_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"ready\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Ready handled", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The max time consumed by raftstore events", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 175, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_raftstore_event_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_raftstore_event_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_broadcast_normal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "broadcast_normal {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999999,(\n sum(rate(\n tikv_broadcast_normal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max duration of raft store events", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Number of messages polled in one batch by the main peer worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 176, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n rfstore_receive_msgs_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n rfstore_receive_msgs_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "rfstore batch message count", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Size of messages polled in one batch by the main peer worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 177, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n rfstore_receive_msgs_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n rfstore_receive_msgs_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "rfstore batch message size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time spent in different steps of the rfstore main worker loop", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 178, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_store_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-handle-store-msg-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_store_msg_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_receive_msgs_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-receive-msgs-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_receive_msgs_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_sync_aux_worker_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-sync-aux-workers-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_sync_aux_worker_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_send_aux_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-send-aux-tasks-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_send_aux_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_process_msgs_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-process-msgs-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n rfstore_process_msgs_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "rfstore main worker time breakdown", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Replica read lock checking duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 179, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_replica_read_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_replica_read_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Replica read lock checking duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The length of peer msgs for each round handling", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 180, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_peer_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_peer_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Peer msg length distribution", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The length of apply msgs for each round handling", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 181, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_raftstore_apply_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_raftstore_apply_msg_len_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply msg length distribution", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of regions that enable apply unpersisted raft log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 182, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_raft_enable_unpersisted_apply_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_raft_enable_unpersisted_apply_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Enable apply unpersisted log region count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of raft logs between apply and persisted index", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 183, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-max", "metric": "", "query": "histogram_quantile(1,(\n sum(rate(\n tikv_raft_apply_ahead_of_persist_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply ahead of persistence raft log count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of each apply future task running in one round", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 184, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_rfstore_apply_running_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-P99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_rfstore_apply_running_duration_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply Poll Handle Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of apply batch a apply future task handles", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 185, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_rfstore_apply_handle_msgs_batch_number_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-P99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_rfstore_apply_handle_msgs_batch_number_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_rfstore_apply_handle_msgs_batch_number_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_rfstore_apply_handle_msgs_batch_number_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-avg", "metric": "", "query": "(sum(rate(\n tikv_rfstore_apply_handle_msgs_batch_number_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_rfstore_apply_handle_msgs_batch_number_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply Poll Handle Batch Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Process", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 186, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Raft messages sent by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 187, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Sent messages per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Raft messages flushed by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 188, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_message_flush_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_message_flush_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flush messages per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of Raft messages received by each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 189, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_message_recv_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_message_recv_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Receive messages per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of different types of Raft messages that are sent", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 190, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status=\"accept\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status=\"accept\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Messages", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of vote messages that are sent in Raft", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 191, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"vote\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"vote\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Vote", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of dropped Raft messages per type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 192, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_dropped_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_dropped_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status=\"drop\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",status=\"drop\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_rfstore_router_raft_message_dropped_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "router_{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_rfstore_router_raft_message_dropped_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft dropped messages", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed waiting to send Raft Messages", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 193, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Raft Message Send Wait duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed waiting to send Raft Messages per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 194, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_raft_message_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_server_raft_message_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"send_wait\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Raft Message Send Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to transmit Raft Messages over the network, reported by the receiver", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 195, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Raft Message Receive Delay duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to transmit Raft Messages over the network per TiKV instance, reported by the receiver", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 196, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_server_raft_message_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_server_raft_message_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_server_raft_message_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_server_raft_message_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"receive_delay\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Raft Message Receive Delay Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Message", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 197, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of admin proposals", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 198, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"conf_change|transfer_leader\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_proposal_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"conf_change|transfer_leader\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Admin proposals", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of the processed apply command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 199, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_admin_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"compact\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_admin_cmd_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"compact\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Admin apply", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of raftstore split checks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 200, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_check_split_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"ignore\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_check_split_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"ignore\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Check split", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when running split check in .9999", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 201, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_check_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_raftstore_check_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% Check split duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 202, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_load_base_split_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(delta(\n tikv_load_base_split_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Load base split event", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 203, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.8,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "80%-{{instance}}", "metric": "", "query": "histogram_quantile(0.8,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_load_base_split_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_load_base_split_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_load_base_split_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_load_base_split_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_load_base_split_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Load base split duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Per-region CPU observed before load-fit filtering when load-base split evaluates regions. Unit follows other CPU panels: 100% means one CPU core. This is pre-load-fit decision input; use the tail distribution to tune split thresholds. Use additional_groupby=instance only when drilling into a specific TiKV; keep it as none for large clusters.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 204, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(histogram_quantile(0.9999,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"cpu_millicores\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) / 1000)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "(histogram_quantile(0.9999,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"cpu_millicores\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) / 1000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(histogram_quantile(0.99,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"cpu_millicores\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) / 1000)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "(histogram_quantile(0.99,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"cpu_millicores\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) / 1000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((sum(rate(\n tikv_load_base_split_region_load_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"cpu_millicores\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_load_base_split_region_load_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"cpu_millicores\"}\n [$__rate_interval]\n)) by ($additional_groupby) ) / 1000)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "((sum(rate(\n tikv_load_base_split_region_load_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"cpu_millicores\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_load_base_split_region_load_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"cpu_millicores\"}\n [$__rate_interval]\n)) by ($additional_groupby) ) / 1000)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Observed region CPU for load-base split", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Per-region QPS observed before load-fit filtering when load-base split evaluates regions. This is pre-load-fit decision input; use the tail distribution to tune split thresholds. Use additional_groupby=instance only when drilling into a specific TiKV; keep it as none for large clusters.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 205, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"qps\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"qps\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"qps\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"qps\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_load_base_split_region_load_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"qps\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_load_base_split_region_load_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"qps\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_load_base_split_region_load_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"qps\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_load_base_split_region_load_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"qps\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_load_base_split_region_load_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"qps\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_load_base_split_region_load_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"qps\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Observed region QPS for load-base split", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "reqps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Per-region read bytes observed before load-fit filtering when load-base split evaluates regions. Unit: KiB. This is pre-load-fit decision input; use the tail distribution to tune split thresholds. Use additional_groupby=instance only when drilling into a specific TiKV; keep it as none for large clusters.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 206, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes_kib\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes_kib\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes_kib\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_load_base_split_region_load_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes_kib\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_load_base_split_region_load_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes_kib\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_load_base_split_region_load_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes_kib\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_load_base_split_region_load_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes_kib\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_load_base_split_region_load_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes_kib\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_load_base_split_region_load_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes_kib\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_load_base_split_region_load_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes_kib\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Observed region read bytes for load-base split", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "kbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 207, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_peer_in_flashback_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_raftstore_peer_in_flashback_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Peer in Flashback State", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Admin", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 208, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 209, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n rfstore_raft_entry_fetches_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n rfstore_raft_entry_fetches_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log fetch", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 210, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/pending-task/", "bars": false, "fill": 1, "fillBelowTo": null, "lines": true, "transform": "negative-Y", "yaxis": 2, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n rfstore_raft_entry_fetches_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n rfstore_raft_entry_fetches_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n rfstore_raft_entry_fetches_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n rfstore_raft_entry_fetches_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}}", "metric": "", "query": "(sum(rate(\n rfstore_raft_entry_fetches_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n rfstore_raft_entry_fetches_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"async-read-worker\"}\n \n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "pending-task-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"async-read-worker\"}\n \n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log async fetch task duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 211, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_raftstore_log_lag_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_raftstore_log_lag_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log lag", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Log", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 212, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The resource usage of Raft Engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 213, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n raft_engine_resouce_usage_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n raft_engine_resouce_usage_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Resource Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average number of raft log entries", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 214, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_total_entries_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} entries {{$additional_groupby}}", "metric": "", "query": "avg((\n raft_engine_total_entries_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Entry Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of operations per second", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 215, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_persist_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_fetch_entries_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_fetch_entries_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Operation", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time used in read/write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 216, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_persist_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_fetch_entries_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_fetch_entries_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read/Write Duration $optional_quantile", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The I/O flow rate", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 217, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_region_write_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_region_write_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "$optional_quantile duration breakdown of write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 218, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_wal_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "wal - $optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_wal_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_wal_write_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "throttling - $optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_wal_write_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "apply - $optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Duration Breakdown $optional_quantile", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The bytes per write", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 219, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_region_write_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n raft_engine_region_write_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_region_write_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n raft_engine_region_write_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n raft_engine_region_write_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_region_write_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n raft_engine_region_write_batch_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n raft_engine_region_write_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_region_write_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_region_write_batch_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Bytes / Written", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "$optional_quantile duration breakdown of WAL write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 220, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_truncate_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "truncate - $optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_truncate_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_rotate_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "rotate - $optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_rotate_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_compact_wal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "compact_wal - $optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n raft_engine_compact_wal_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Duration of WAL Operation - $optional_quantile", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average number of files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 221, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_pending_compaction_wals\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "pending_compaction {{$additional_groupby}}", "metric": "", "query": "avg((\n raft_engine_pending_compaction_wals\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_total_wals_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total {{$additional_groupby}}", "metric": "", "query": "avg((\n raft_engine_total_wals_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "File Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The $optional_quantile duration of operations of raft engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 222, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_take_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "snapshot - $optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_take_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_backup_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "backup-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n raft_engine_backup_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Other Durations $optional_quantile", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The size of GCed raft log", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 223, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n raft_engine_log_file_gc_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (status, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{status}} {{$additional_groupby}}", "metric": "", "query": "avg((\n raft_engine_log_file_gc_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (status, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Log GC Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average number of raft engine backup", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 224, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n raft_engine_backup_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (status, $additional_groupby) > -10", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{status}} {{$additional_groupby}}", "metric": "", "query": "sum(delta(\n raft_engine_backup_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (status, $additional_groupby) > -10", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": null, "mappings": [ { "options": { "0": { "color": "red", "index": null, "text": "Unhealthy" }, "1": { "color": "green", "index": null, "text": "Healthy" } }, "type": "value" } ], "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 225, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n raft_engine_dfs_worker_healthy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "((\n raft_engine_dfs_worker_healthy\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "DFS Worker Status", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total number of bytes uploaded to DFS", "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": null, "mappings": [], "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 226, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n raft_engine_dfs_uploaded_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "((\n raft_engine_dfs_uploaded_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Total Uploaded Bytes to DFS", "transformations": [], "transparent": false, "type": "stat" }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of DFS requests issued by rfengine.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 227, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_dfs_requests_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_dfs_requests_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "DFS Request Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The byte rates of uploading WAL to DFS.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 228, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n raft_engine_dfs_uploaded_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n raft_engine_dfs_uploaded_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "DFS Upload Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Engine", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 229, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memory usage of Kv Engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 230, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n kv_engine_mem_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",db=\"kv\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "max((\n kv_engine_mem_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",db=\"kv\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n kv_engine_mem_size_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "memtable-quota {{$additional_groupby}}", "metric": "", "query": "max((\n kv_engine_mem_size_quota_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n kv_engine_dfs_load_memory_usage_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "dfs_load {{$additional_groupby}}", "metric": "", "query": "max((\n kv_engine_dfs_load_memory_usage_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n kv_engine_ia_manager_segments_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "segment {{$additional_groupby}}", "metric": "", "query": "max((\n kv_engine_ia_manager_segments_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of get operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 231, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_get_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "get {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_get_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Get operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing get operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 232, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_get_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_get_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n kv_engine_get_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n kv_engine_get_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n kv_engine_get_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n kv_engine_get_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Get duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of seek operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 233, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_seek_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_seek_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Seek operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing seek operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 234, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_seek_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_seek_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n kv_engine_seek_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n kv_engine_seek_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n kv_engine_seek_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n kv_engine_seek_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Seek duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of write operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 235, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing write operation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 236, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n kv_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n kv_engine_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n kv_engine_write_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "avg((\n kv_engine_write_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The bytes flow of write operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 237, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The bytes per write", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 238, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"bytes\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Bytes / Write", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The bytes flow of write operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 239, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keys flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of keys per write", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 240, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_write_flow_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_write_flow_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_write_flow_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"keys\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keys / Write", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The lookup and miss QPS of block cache", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 241, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_cache_lookup\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "lookup-{{instance}}", "metric": "", "query": "sum(rate(\n kv_engine_cache_lookup\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "miss-{{instance}}", "metric": "", "query": "sum(rate(\n kv_engine_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block Cache QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The ratio of block cache misses to lookups", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 242, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n kv_engine_cache_lookup\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "miss rate-{{instance}}", "metric": "", "query": "(sum(rate(\n kv_engine_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n kv_engine_cache_lookup\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Block Cache Miss Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of shards in KV Engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 243, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_shards_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_shards_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_initial_flushed_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "initial_flushed {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_initial_flushed_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_active_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "active {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_active_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_compacting_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "compacting {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_compacting_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_pending_compaction_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "pending_compaction {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_pending_compaction_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_has_del_prefixes_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "has_del_prefixes {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_has_del_prefixes_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_inner_key_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "inner_key {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_inner_key_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_ready_destroy_range_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "ready_destroy_range {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_ready_destroy_range_shards\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Shards", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of files in KV Engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 244, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_cf_level_num_files\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (cf, level, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{level}} {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_cf_level_num_files\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (cf, level, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Files Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The size of files in KV Engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 245, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_cf_level_total_sizes_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (cf, level, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{level}} {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_cf_level_total_sizes_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (cf, level, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Files Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of memtables", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 246, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_mem_tables_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": " {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_mem_tables_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memtable Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The size of memtables", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 247, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_mem_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",db=\"kv\",type=\"memtable\"}\n \n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_mem_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",db=\"kv\",type=\"memtable\"}\n \n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memtable Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of L0 tables", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 248, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_l0_tables_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": " {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_l0_tables_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "L0 Table Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The size of L0 tables", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 249, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_l0_tables_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_l0_tables_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "L0 Table Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of blob tables", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 250, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_blob_tables_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": " {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_blob_tables_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob Table Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The size of blob tables", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 251, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_in_use_blob_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "in_use-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_in_use_blob_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_total_blob_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_total_blob_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Blob Table Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of partial files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 252, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_partial_l0_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "partial_l0 {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_partial_l0_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_partial_blob_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "partial_blob {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_partial_blob_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_partial_ln_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "partial_ln {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_partial_ln_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Partial File Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The size of index and filter files", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 253, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_index_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "index {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_index_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_filter_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "filter {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_filter_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Index & Filter Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of entries", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 254, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_entries\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "entries {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_entries\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_old_entries\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "old_entries {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_old_entries\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_tombstones\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "tombstones {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_tombstones\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Entries Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The throughput of DFS operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 255, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_dfs_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",dfs_type=\"s3\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_dfs_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",dfs_type=\"s3\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "DFS Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The throughput of load file at each level", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 256, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_level_write_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{level}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_level_write_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (level, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Level Write Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The OPS of DFS operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 257, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_dfs_request_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_dfs_request_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "DFS Request Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The OPS of DFS request retries", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 258, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_dfs_retry_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_dfs_retry_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "DFS Request Retry Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of DFS operations in ms", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 105 }, "height": null, "hideTimeOverride": false, "id": 259, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_dfs_latency_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_dfs_latency_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_dfs_latency_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_dfs_latency_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_dfs_latency_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_dfs_latency_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_dfs_latency_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_dfs_latency_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_dfs_latency_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_dfs_latency_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "DFS Request Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of DFS operations including retry times in ms", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 105 }, "height": null, "hideTimeOverride": false, "id": 260, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_dfs_latency_with_retry_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "DFS Request Duration with Retry", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Wait duration when acquiring DFS load memory (only recorded when memory is not immediately available)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 112 }, "height": null, "hideTimeOverride": false, "id": 261, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_dfs_load_memory_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "DFS Load Memory Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The file load statistic when preparing changeset", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 112 }, "height": null, "hideTimeOverride": false, "id": 262, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_prepare_load_remote_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "s3 {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_prepare_load_remote_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_prepare_use_local_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "local {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_prepare_use_local_file\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Prepare ChangeSet File Load Source", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Count of CacheFS hit", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 119 }, "height": null, "hideTimeOverride": false, "id": 263, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_cachefs_req_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"hit\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}}", "metric": "", "query": "sum(rate(\n kv_engine_cachefs_req_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"hit\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CacheFS Hit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "CacheFS hit rate", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 119 }, "height": null, "hideTimeOverride": false, "id": 264, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_cachefs_req_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"hit\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n kv_engine_cachefs_req_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "rate-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_cachefs_req_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"hit\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n kv_engine_cachefs_req_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CacheFS Hit Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate of duplicated changesets detected by KvEngine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 119 }, "height": null, "hideTimeOverride": false, "id": 265, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_duplicated_changeset_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "duplicated changeset {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_duplicated_changeset_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Duplicated ChangeSet Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The distribution of ingestion levels (0-3) in the storage engine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 126 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 266, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n kv_engine_ingest_level_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n kv_engine_ingest_level_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Ingest Level Distribution", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate of ingestion operations by level", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 126 }, "height": null, "hideTimeOverride": false, "id": 267, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_ingest_level_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total ingestion rate {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_ingest_level_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Ingest Level Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Kv Engine", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 268, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Bytes processed by compaction (input KV / output compressed), broken down by type, kind, and format", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 269, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_compaction_process_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, kind, format, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{kind}}-{{format}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_compaction_process_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, kind, format, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Process Bytes Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rows processed by compaction (input/output), broken down by type, kind, and format", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 270, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_compaction_process_rows_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, kind, format, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{kind}}-{{format}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_compaction_process_rows_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, kind, format, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Process Rows Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Per-task compaction input throughput (bytes/sec), by type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 271, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_compaction_task_handled_bytes_throughput_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_compaction_task_handled_bytes_throughput_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_compaction_task_handled_bytes_throughput_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_compaction_task_handled_bytes_throughput_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_compaction_task_handled_bytes_throughput_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n kv_engine_compaction_task_handled_bytes_throughput_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_compaction_task_handled_bytes_throughput_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n kv_engine_compaction_task_handled_bytes_throughput_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_compaction_task_handled_bytes_throughput_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_compaction_task_handled_bytes_throughput_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Task Input Bytes Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Per-task compaction input throughput (rows/sec), by type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 272, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_compaction_task_handled_rows_throughput_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_compaction_task_handled_rows_throughput_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_compaction_task_handled_rows_throughput_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_compaction_task_handled_rows_throughput_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_compaction_task_handled_rows_throughput_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n kv_engine_compaction_task_handled_rows_throughput_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_compaction_task_handled_rows_throughput_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n kv_engine_compaction_task_handled_rows_throughput_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_compaction_task_handled_rows_throughput_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_compaction_task_handled_rows_throughput_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Task Input Rows Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of compaction results (success/failure)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 273, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_compaction_result_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{result}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_compaction_result_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Result Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of compaction triggers, by type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 274, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_compaction_trigger_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_compaction_trigger_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Trigger Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of remote compaction requests that exceed the memory limit", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 275, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_remote_compact_exceed_memory_limit_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "exceed memory limit {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_remote_compact_exceed_memory_limit_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Compaction Exceed Memory Limit Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Kv Engine - Compaction", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 276, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 277, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n kv_engine_ia_manager_segments_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "used-{{instance}}", "metric": "", "query": "max((\n kv_engine_ia_manager_segments_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n kv_engine_ia_small_queue_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "cap-{{instance}}", "metric": "", "query": "max((\n kv_engine_ia_small_queue_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA Segment Memory Cache", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 278, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n kv_engine_ia_manager_segments_disk_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "used-{{instance}}", "metric": "", "query": "max((\n kv_engine_ia_manager_segments_disk_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n kv_engine_ia_main_queue_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "cap-{{instance}}", "metric": "", "query": "max((\n kv_engine_ia_main_queue_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA Segment Disk Cache", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Count of IA segment hit", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 279, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_ia_remote_read_segment_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}}", "metric": "", "query": "sum(rate(\n kv_engine_ia_remote_read_segment_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA Segment Cache Hit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "IA Segment hit rate", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 280, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_ia_remote_read_segment_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / (sum(rate(\n kv_engine_ia_remote_read_segment_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) + sum(rate(\n kv_engine_ia_remote_read_segment_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "rate-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_ia_remote_read_segment_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / (sum(rate(\n kv_engine_ia_remote_read_segment_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) + sum(rate(\n kv_engine_ia_remote_read_segment_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA Segment Cache Hit Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Throughput through IA IO rate limiter and configured cap. Cap metrics are per TiKV process, summed by this panel; defaults are 500 MiB/s per process, and 0 means unlimited.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 281, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_ia_io_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_ia_io_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_ia_io_throughput_cap\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "cap-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_ia_io_throughput_cap\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA IO Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time spent throttling IA remote reads and local writes.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 282, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_ia_io_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_ia_io_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_ia_io_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_ia_io_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_ia_io_throttle_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n kv_engine_ia_io_throttle_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_ia_io_throttle_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n kv_engine_ia_io_throttle_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_ia_io_throttle_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_ia_io_throttle_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA IO Throttle Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Pending and inflight storage-class transition regions. Limits are per TiKV process.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 283, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_ia_transition_region_pending\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "pending {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_ia_transition_region_pending\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n kv_engine_ia_transition_region_inflight\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "inflight {{$additional_groupby}}", "metric": "", "query": "sum((\n kv_engine_ia_transition_region_inflight\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage Class Transition Regions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of storage-class transition region completions by result.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 284, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_ia_transition_region_finished_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{result}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_ia_transition_region_finished_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage Class Transition Region Result Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Remote read bytes admitted by the IA to Standard transition limiter. Bytes are based on DFS file size before each remote read starts.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 285, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_ia_to_standard_transition_remote_read_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "admitted {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_ia_to_standard_transition_remote_read_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA to Standard Remote Read Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of time spent throttling IA to Standard transition remote reads.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 286, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_ia_to_standard_transition_remote_read_throttle_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "throttled {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_ia_to_standard_transition_remote_read_throttle_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA to Standard Remote Read Throttle Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time taken by KvEngine to complete read a IA segment", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 287, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "IA Segmenet Read Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time taken by KvEngine to complete read a IA segment", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 288, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{instance}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_ia_remote_read_segment_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n kv_engine_ia_remote_read_segment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{instance}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_ia_remote_read_segment_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) / sum(rate(\n kv_engine_ia_remote_read_segment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_ia_remote_read_segment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_ia_remote_read_segment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% IA segment read duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Storage size used by each storage class (IA and Standard)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 289, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"used\"}\n \n)) by (storage_class, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{storage_class}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_store_size_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"used\"}\n \n)) by (storage_class, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage Size by Storage Class", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "IA", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 290, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU usage of each copr worker instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 291, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "quota-{{instance}}", "metric": "", "query": "(((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max-quota", "metric": "", "query": "max((((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) ))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "min((((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "min-quota", "metric": "", "query": "min((((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) ))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Copr Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memory usage per copr worker instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 292, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) > 0)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "quota-{{instance}}", "metric": "", "query": "(((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) > 0)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) > 0))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max-quota", "metric": "", "query": "max((((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) > 0))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "min((((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) > 0))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "min-quota", "metric": "", "query": "min((((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"coprocessor-worker\"}\n \n)) > 0))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Copr Worker Memory", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU usage of each TiKV worker instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 293, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n process_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "quota-{{instance}}", "metric": "", "query": "(((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max-quota", "metric": "", "query": "max((((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) ))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "min((((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) ))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "min-quota", "metric": "", "query": "min((((\n container_spec_cpu_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) / ((\n container_spec_cpu_period\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) ))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiKV Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memory usage per TiKV worker instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 294, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) > 0)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "quota-{{instance}}", "metric": "", "query": "(((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) > 0)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) > 0))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max-quota", "metric": "", "query": "max((((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) > 0))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "min((((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) > 0))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "min-quota", "metric": "", "query": "min((((\n container_spec_memory_limit_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\"tikv-worker\"}\n \n)) > 0))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiKV Worker Memory", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle remote compaction requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 295, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_worker_remote_compact_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_worker_remote_compact_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Compaction Request duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle remote compaction requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 296, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_compact_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compaction Request duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of remote compaction or coprocessor requests are under processing", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 297, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_worker_remote_compact_processing_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "compact-processing {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_worker_remote_compact_processing_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_worker_remote_cop_processing_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "copr-processing {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_worker_remote_cop_processing_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Processing Requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of remote requests that failed, by low-cardinality failure reason", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 298, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_compact_failed_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "compact-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_compact_failed_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_cop_failed_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "copr-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_cop_failed_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Requests Failures", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of remote requests being waiting, limited by concurrency limiter", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 299, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_worker_limiter_waiting_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",instance=~\"$worker_instance\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_worker_limiter_waiting_requests_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",instance=~\"$worker_instance\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Concurrency Limiter Waiting Length", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The waiting duration of handling remote requests limited by concurrency limiter", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 300, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_worker_limiter_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} - $optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_worker_limiter_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Concurrency Limiter Waiting Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle remote copr snapshot duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 301, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Snapshot Request duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle remote copr snapshot duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 302, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_cop_snapshot_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot Request duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle remote copr process duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 303, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_worker_remote_cop_process_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_worker_remote_cop_process_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Copr Request Process duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle remote copr process duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 304, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_cop_process_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Copr Request Process duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle remote coprocessor read requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 305, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_worker_remote_cop_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_worker_remote_cop_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Copr Request duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle remote coprocessor read requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 306, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_cop_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Copr Request duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The QPS of completed remote coprocessor requests, including successful and failed outcomes, by top-level request type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 307, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_cop_requests_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\",outcome=~\"success|failure\"}\n [$__rate_interval]\n)) by (request_type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{request_type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_cop_requests_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\",outcome=~\"success|failure\"}\n [$__rate_interval]\n)) by (request_type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Copr Request Operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The size of successful remote coprocessor responses", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 308, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_cop_dag_response_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "dag {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_cop_dag_response_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_analyze_response_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "analyze {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_analyze_response_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_checksum_response_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "checksum {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_checksum_response_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Copr Response Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Processed and total bytes scanned by completed remote coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 309, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_cop_scan_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (kind, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{kind}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_cop_scan_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (kind, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Copr Scan Bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Processed and total keys scanned by completed remote coprocessor requests, split by request kind", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 310, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_remote_cop_scan_keys_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (request_kind, kind, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{request_kind}}-{{kind}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_remote_cop_scan_keys_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (request_kind, kind, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Copr Scan Keys", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of bytes are currently being used by requests limited by memory limiter", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 311, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_worker_memory_limiter_current_used\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",instance=~\"$worker_instance\"}\n \n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_worker_memory_limiter_current_used\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",instance=~\"$worker_instance\"}\n \n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Limiter Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "P99 duration of remote coprocessor prefetch", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 312, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_prefetch_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "prefetch {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_prefetch_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiKV/Cop Worker Duration (P99)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "P99 cache hit percentage of remote coprocessor prefetch", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 313, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_prefetch_cache_hit_percent_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "prefetch cache hit {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_worker_remote_cop_prefetch_cache_hit_percent_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cop Worker IA Prefetch Cache Hit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percent", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Cloud Worker Service", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 314, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 315, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_worker_native_br_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_worker_native_br_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_worker_native_br_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_worker_native_br_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_worker_native_br_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_worker_native_br_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_worker_native_br_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_worker_native_br_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Native BR Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 316, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_native_br_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_native_br_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Native BR RPC Operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of successful and failed backup operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 317, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n native_br_backup_success\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "success-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n native_br_backup_success\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n native_br_backup_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "error-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n native_br_backup_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Native BR Backup Operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of errors during restoration by type", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 318, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n native_br_restore_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n native_br_restore_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n native_br_restore_rfengine_wal_epoch_overwritten_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "wal_epoch_overwritten-{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n native_br_restore_rfengine_wal_epoch_overwritten_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Native BR Restore Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total bytes of restored snapshots", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 319, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n native_br_restored_snapshot_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n native_br_restored_snapshot_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Native BR Restored Snapshot Bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of restore snapshot of a shard", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 320, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n native_br_restore_snapshot_request_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n native_br_restore_snapshot_request_secs_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Native BR Restore Snapshot Request Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Pending and restored keyspace data sizes", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 321, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/.*rate/", "bars": false, "fill": 1, "fillBelowTo": null, "lines": true, "yaxis": 2, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n native_br_restore_pending_data_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Pending data size", "metric": "", "query": "sum((\n native_br_restore_pending_data_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n native_br_restored_data_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Restored data size rate", "metric": "", "query": "sum(rate(\n native_br_restored_data_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n native_br_restored_kv_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Restored kv size rate", "metric": "", "query": "sum(rate(\n native_br_restored_kv_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n native_br_restored_data_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Restored data size (total)", "metric": "", "query": "sum((\n native_br_restored_data_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n native_br_restored_kv_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Restored kv size (total)", "metric": "", "query": "sum((\n native_br_restored_kv_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Restore keyspace bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Cloud Worker - Native BR", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 322, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Schema manager sync loops and errors per minute", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 323, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_worker_schema_manager_sync_loop_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [1m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "loops - {{instance}}", "metric": "", "query": "sum(increase(\n tikv_worker_schema_manager_sync_loop_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [1m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_worker_schema_manager_sync_loop_error_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [1m]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "error ({{type}}) - {{instance}}", "metric": "", "query": "sum(increase(\n tikv_worker_schema_manager_sync_loop_error_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$worker_instance\"}\n [1m]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Schema manager loops", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Cloud Worker - Schema Manager", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 324, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of files on disk", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 325, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n rfstore_local_file_on_disk\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n rfstore_local_file_on_disk\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "On Disk Files", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of alive files used by kvengine", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 326, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n rfstore_local_file_alive\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n rfstore_local_file_alive\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Alive Files", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of files garbage collected", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 327, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n rfstore_local_file_gc_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n rfstore_local_file_gc_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC Files", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of files skipped by GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 328, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n rfstore_local_file_skip_gc\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n rfstore_local_file_skip_gc\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Skip GC Files", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of files pending for garbage collection", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 329, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n rfstore_local_file_pending_gc\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n rfstore_local_file_pending_gc\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Pending GC Files", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of local file GC errors", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 330, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n rfstore_local_file_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n rfstore_local_file_error_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Local File GC", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 331, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Counter when throttle happens per limiter", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 332, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_limiter_throttle_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (limiter, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{limiter}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_limiter_throttle_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (limiter, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Throttling counter", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Counter when different throttling actions happen per limiter", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 333, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_limiter_throttle_action_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (limiter, type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{limiter}}-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_limiter_throttle_action_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (limiter, type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Throttling algorithm actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 334, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_write_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) != 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "throttle-{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_throttle_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) != 0", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 335, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_scheduler_throttle_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Throttle duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Flow Control", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 336, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of commands on each stage", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 337, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{stage}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler stage total", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of different priority commands", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 338, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (priority, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{priority}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_commands_pri_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (priority, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler priority commands", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of pending commands per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 339, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_context_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_context_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler pending commands", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of tasks being executed in the scheduler pool (excludes tasks not yet polled)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 340, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler running tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total writing bytes of commands on each stage", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 341, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_writing_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler writing bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of bytes used by scheduler", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 342, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"in_use\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"in_use\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"capacity\"}\n \n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_scheduler_memory_quota_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"capacity\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler memory quota", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time tasks wait to be scheduled (YATP or Tokio backend)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 343, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-yatp-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-tokio-{{instance}}", "metric": "", "query": "((\n tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler Pool Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Async resolve lock task dispatch, inqueue, and dequeue throughput.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 344, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_async_resolve_lock_dispatch_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "dispatch-{{result}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_async_resolve_lock_dispatch_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_async_resolve_lock_inqueue_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "inqueue-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_async_resolve_lock_inqueue_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_async_resolve_lock_dequeue_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "dequeue-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_async_resolve_lock_dequeue_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Async resolve lock queue flow OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current size of async resolve lock pending, inflight, and blocked queues.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 345, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_scheduler_async_resolve_lock_queue_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_scheduler_async_resolve_lock_queue_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Async resolve lock queue size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Async resolve lock background scheduled task QPS, scheduled txn-status QPS, and dropped task QPS.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 346, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/txn/", "bars": false, "fill": 1, "fillBelowTo": null, "lines": true, "yaxis": 2, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_async_resolve_lock_bg_schedule_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_async_resolve_lock_bg_schedule_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Async resolve lock background schedule QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Scheduler", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 347, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time from task spawn to first poll", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 348, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-yatp-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-tokio-{{instance}}", "metric": "", "query": "((\n tikv_scheduler_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Schedule Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of tasks being executed in the scheduler pool (excludes tasks not yet polled)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 349, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(avg_over_time(\n tikv_scheduler_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum(avg_over_time(\n tikv_scheduler_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Running Tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time tasks spend executing (being polled)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 350, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-yatp-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sched-yatp-pool.*\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_scheduler_tokio_pool_mean_poll_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-tokio-{{instance}}", "metric": "", "query": "((\n tikv_scheduler_tokio_pool_mean_poll_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Poll Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time tasks wait to be polled after being woken", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 351, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_scheduler_tokio_pool_mean_scheduled_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-tokio-{{instance}}", "metric": "", "query": "((\n tikv_scheduler_tokio_pool_mean_scheduled_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Scheduled Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time tasks spend idle waiting on external events (I/O)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 352, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_scheduler_tokio_pool_mean_idle_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-tokio-{{instance}}", "metric": "", "query": "((\n tikv_scheduler_tokio_pool_mean_idle_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Idle Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of tasks polled for first time", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 353, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_tokio_pool_first_poll_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "tokio-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_tokio_pool_first_poll_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "First Poll Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Scheduler Worker Pool", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 354, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The execution time used by each level in YATP thread pools per second. Level 0 refers to small queries.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 355, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{level}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_multilevel_level_elapsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Time Used by Level", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The chance that level 0 (small) tasks are scheduled in YATP thread pools.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 356, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{instance}}", "metric": "", "query": "((\n tikv_multilevel_level0_chance\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Level 0 Chance", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time a completed task spent waiting in a YATP thread pool.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 357, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{name}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{name}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_wait_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) / sum(rate(\n tikv_yatp_task_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{name}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) / sum(rate(\n tikv_yatp_task_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_yatp_task_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total execution time across all polls of a completed YATP task.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 358, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{name}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{name}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_exec_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_exec_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) / sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{name}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_exec_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) / sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_yatp_task_exec_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Execute Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Execution time of a single YATP task poll at each level.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 359, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{name}}-{{level}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{name}}-{{level}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_poll_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, $additional_groupby) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{name}}-{{level}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, $additional_groupby) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{name}}-{{level}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, level, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Duration of One Time Slice", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of times a completed YATP task was polled before completion.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 360, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{name}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{name}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_task_execute_times_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_execute_times_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) / sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{name}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_execute_times_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) / sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_yatp_task_execute_times_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Schedule Times", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "YATP", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 361, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of commands on each stage in commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 362, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "busy-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_too_busy_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{stage}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_stage_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (stage, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler stage total", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 363, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_scheduler_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler command duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time which is caused by latch wait in commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 364, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_latch_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler latch wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of keys read by a commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 365, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_read_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_kv_command_key_read_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_scheduler_kv_command_key_read_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_command_key_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler keys read", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of keys written by a commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 366, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_kv_command_key_write_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_kv_command_key_write_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_scheduler_kv_command_key_write_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_command_key_write_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler keys written", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of each CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 367, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of lock CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 368, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"lock\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"lock\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details [lock]", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of write CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 369, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"write\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"write\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details [write]", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys scan details of default CF when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 370, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"default\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_kv_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=\"$command\", cf=\"default\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler scan details [default]", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on reading when executing commit command", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 371, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_scheduler_processing_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_scheduler_processing_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_scheduler_processing_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_processing_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler command read duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on checking memory locks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 372, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_storage_check_mem_lock_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_storage_check_mem_lock_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Check memory locks duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null } ], "repeat": "command", "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Scheduler - $command", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 373, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle coprocessor read requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 374, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Request duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle coprocessor read requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 375, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Average coprocessor CPU cores used by each keyspace over the recent window", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 376, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_coprocessor_keyspace_handle_cpu_cores\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{keyspace_id}}", "metric": "", "query": "sum((\n tikv_coprocessor_keyspace_handle_cpu_cores\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor CPU Usage per Keyspace", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 377, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 378, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (reason, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{reason}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (reason, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Request Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 379, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "KV Cursor Operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 380, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_scan_keys_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_scan_keys_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_keys_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "KV Cursor Operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 381, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "delete_skipped-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_rocksdb_perf\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",metric=\"internal_delete_skipped_count\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total RocksDB Perf Statistics", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 382, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Response Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total bytes of memory used by coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 383, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n tikv_coprocessor_memory_quota\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Quota", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor Overview", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 384, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when handling coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 385, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Handle duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed to handle coprocessor requests per TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 386, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}}-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (req, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Handle duration by store", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when coprocessor requests are wait for being handled", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 387, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when coprocessor requests are wait for being handled in each TiKV instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 388, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_wait_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}}-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_wait_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) / sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_wait_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"all\"}\n [$__rate_interval]\n)) by (req, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Wait duration by store", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 389, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (vec_type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{vec_type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_dag_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (vec_type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total DAG Requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of DAG executors", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 390, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total DAG Executors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The QPS of remote execution decisions by primary reason: size/range eligibility, long-query or lazy-pattern eligibility, load-aware force thresholds, or CPU state", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 391, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_remote_decision_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",result=\"remote\"}\n [$__rate_interval]\n)) by (reason, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{reason}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_remote_decision_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",result=\"remote\"}\n [$__rate_interval]\n)) by (reason, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Copr Offload Reasons", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 392, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details (Table Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 393, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details (Index Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 394, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"select|select_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details by CF (Table Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 395, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_scan_details\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"index|index_by_in_memory_engine\"}\n [$__rate_interval]\n)) by (cf, tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total Ops Details by CF (Index Scan)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on checking memory locks for coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 396, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Memory lock checking duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed on checking memory locks for coprocessor requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 397, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_mem_lock_check_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory lock checking duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor Detail", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 398, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time from task spawn to first poll", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 399, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-yatp-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_pool_schedule_wait_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_pool_schedule_wait_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-tokio-{{instance}}", "metric": "", "query": "((\n tikv_tokio_pool_mean_first_poll_delay_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Schedule Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of concurrently running tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 400, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(avg_over_time(\n tikv_unified_read_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (priority, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{priority}} {{$additional_groupby}}", "metric": "", "query": "sum(avg_over_time(\n tikv_unified_read_pool_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (priority, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Running Tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time tasks spend executing (being polled)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 401, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-yatp-{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_task_poll_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_yatp_task_poll_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_tokio_pool_mean_poll_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-tokio-{{instance}}", "metric": "", "query": "((\n tikv_tokio_pool_mean_poll_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Poll Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time tasks wait to be polled after being woken", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 402, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_tokio_pool_mean_scheduled_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-tokio-{{instance}}", "metric": "", "query": "((\n tikv_tokio_pool_mean_scheduled_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Scheduled Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time tasks spend idle waiting on external events (I/O)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 403, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_tokio_pool_mean_idle_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mean-tokio-{{instance}}", "metric": "", "query": "((\n tikv_tokio_pool_mean_idle_duration_seconds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Idle Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of tasks polled for first time", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 404, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_tokio_pool_first_poll_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "tokio-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_tokio_pool_first_poll_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=\"unified-read-pool\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "First Poll Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "CPU and wall time consumed by YATP read-pool task polls for each keyspace. 1.0 means one second per second.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 405, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "cpu-KSID-{{keyspace_id}}", "metric": "", "query": "sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_pool_keyspace_task_wall_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "wall-KSID-{{keyspace_id}}", "metric": "", "query": "sum(rate(\n tikv_yatp_pool_keyspace_task_wall_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keyspace Task CPU / Wall Time Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Duration from pushing a task slice into the YATP keyspace fair queue to popping it for each keyspace.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 406, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-KSID-{{keyspace_id}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-KSID-{{keyspace_id}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-KSID-{{keyspace_id}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-KSID-{{keyspace_id}}", "metric": "", "query": "sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keyspace Fair Queue Slice Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Per-keyspace queued task slices in the YATP keyspace fair queue before a task slice is pushed.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 407, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-KSID-{{keyspace_id}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-KSID-{{keyspace_id}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_length_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_length_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-KSID-{{keyspace_id}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_length_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_length_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_length_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-KSID-{{keyspace_id}}", "metric": "", "query": "sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_length_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keyspace Fair Queue Length", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Number of successful task-slice pops after a task slice is pushed and before it is popped from the YATP keyspace fair queue. Multiple selected keyspaces are aggregated in this heatmap.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 408, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_waits_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_waits_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Keyspace Fair Queue Task Slice Waits", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Average CPU and wall time consumed by each popped YATP read-pool task slice for each keyspace.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 409, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ) * 1000)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "cpu-KSID-{{keyspace_id}}", "metric": "", "query": "((sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ) * 1000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((sum(rate(\n tikv_yatp_pool_keyspace_task_wall_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ) * 1000)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "wall-KSID-{{keyspace_id}}", "metric": "", "query": "((sum(rate(\n tikv_yatp_pool_keyspace_task_wall_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ) * 1000)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keyspace Avg Task CPU / Wall Time Per Slice", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current number of keyspace groups in the YATP keyspace fair queue.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 410, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_yatp_pool_keyspace_fair_queue_group_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "max((\n tikv_yatp_pool_keyspace_fair_queue_group_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Active Fair Keyspace Groups", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "P99 CPU time spent in one YATP read-pool task slice for each keyspace.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 411, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-KSID-{{keyspace_id}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-KSID-{{keyspace_id}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-KSID-{{keyspace_id}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-KSID-{{keyspace_id}}", "metric": "", "query": "sum(rate(\n tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keyspace Task Slice CPU Time P99", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "P99 interval from the later of the previous pop and current enqueue to the current task-slice pop for each keyspace.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 412, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_schedule_interval_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-KSID-{{keyspace_id}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_schedule_interval_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_schedule_interval_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-KSID-{{keyspace_id}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_schedule_interval_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_schedule_interval_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_schedule_interval_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-KSID-{{keyspace_id}}", "metric": "", "query": "(sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_schedule_interval_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) / sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_schedule_interval_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_schedule_interval_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-KSID-{{keyspace_id}}", "metric": "", "query": "sum(rate(\n tikv_yatp_pool_keyspace_fair_queue_task_slice_schedule_interval_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"unified-read.*\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keyspace Task Slice Schedule Interval P99", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Histogram of entries examined during EXTRA_CF conflict checks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 413, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_extra_cf_scan_iterations_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_extra_cf_scan_iterations_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_extra_cf_scan_iterations_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_extra_cf_scan_iterations_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_extra_cf_scan_iterations_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_extra_cf_scan_iterations_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_extra_cf_scan_iterations_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_storage_extra_cf_scan_iterations_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_extra_cf_scan_iterations_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_extra_cf_scan_iterations_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "EXTRA CF Scan Examined Entries", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Unified Read Pool", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 414, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of GC tasks processed by gc_worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 415, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_tasks_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-{{task}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_tasks_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_gc_skipped_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "skipped-{{task}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_gc_skipped_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_task_fail_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "failed-{{task}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_task_fail_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (task, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing GC tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 416, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_gcworker_gc_task_duration_vec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_gcworker_gc_task_duration_vec_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_gcworker_gc_task_duration_vec_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_task_duration_vec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"$command\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC tasks duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The GC duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 417, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1,(\n sum(rate(\n tidb_tikvclient_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (instance, stage, keyspace_name, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{keyspace_name}}-{{stage}}", "metric": "", "query": "histogram_quantile(1,(\n sum(rate(\n tidb_tikvclient_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (instance, stage, keyspace_name, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiDB GC seconds", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of TiDB GC worker actions", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 418, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_tikvclient_gc_worker_actions_total\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (type, keyspace_name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{keyspace_name}}-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tidb_tikvclient_gc_worker_actions_total\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n [$__rate_interval]\n)) by (type, keyspace_name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiDB GC worker actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Progress of ResolveLocks, the first phase of GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 419, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_tikvclient_range_task_stats\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=~\"resolve-locks.*\"}\n \n)) by (result, type, keyspace_name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{keyspace_name}} {{type}} {{result}} {{$additional_groupby}}", "metric": "", "query": "max((\n tidb_tikvclient_range_task_stats\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=~\"resolve-locks.*\"}\n \n)) by (result, type, keyspace_name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "ResolveLocks Progress", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Progress of TiKV's GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 420, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum((\n tikv_gcworker_autogc_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"scan\"}\n \n)) by (instance) / sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n \n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum((\n tikv_gcworker_autogc_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"scan\"}\n \n)) by (instance) / sum((\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n \n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiKV Auto GC Progress", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "keys / second", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 421, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_mvcc_gc_delete_versions_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_keys/s {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_mvcc_gc_delete_versions_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The GC safe points that has been fetched by TiKV", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 422, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_gcworker_autogc_safe_point\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n \n)) by (instance, keyspace_name, $additional_groupby) / (2^18) != 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-null_keyspace {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_gcworker_autogc_safe_point\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n \n)) by (instance, keyspace_name, $additional_groupby) / (2^18) != 0", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_raftstore_keyspace_gc_safe_points\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n \n)) by (instance, keyspace_name, $additional_groupby) / (2^18) != 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{keyspace_name}} {{$additional_groupby}}", "metric": "", "query": "max((\n tikv_raftstore_keyspace_gc_safe_points\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\"}\n \n)) by (instance, keyspace_name, $additional_groupby) / (2^18) != 0", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiKV Side GC Safe Points", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "dateTimeAsIso", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The lifetime of TiDB GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 423, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=\"tikv_gc_life_time\"}\n \n)) by (keyspace_name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{keyspace_name}}", "metric": "", "query": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=\"tikv_gc_life_time\"}\n \n)) by (keyspace_name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC lifetime", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The interval of TiDB GC", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 424, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": false, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=\"tikv_gc_run_interval\"}\n \n)) by (keyspace_name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{keyspace_name}}", "metric": "", "query": "max((\n tidb_tikvclient_gc_config\n {k8s_cluster=\"$k8s_cluster\",sharedpool_id=\"$tidb_cluster\",type=\"tikv_gc_run_interval\"}\n \n)) by (keyspace_name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC interval", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Keys handled in GC compaction filter", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 425, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_filtered {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filtered\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_skip\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_skipped {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_skip\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_mvcc_rollback\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-rollback/mvcc-lock {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_mvcc_rollback\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_orphan_versions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_orphan-versions {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_orphan_versions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_perform\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_performed-times {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_perform\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_failure-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_met\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-deletion-met {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_met\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_handled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-deletion-handled {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_handled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_wasted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}_mvcc-deletion-wasted {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gc_compaction_filter_mvcc_deletion_wasted\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (key_mode, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC in Compaction Filter", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "GC scan write details", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 426, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"write\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}-{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"write\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC scan write details", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "GC scan default details", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 427, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"default\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{key_mode}}-{{tag}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_gcworker_gc_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"default\"}\n [$__rate_interval]\n)) by (key_mode, tag, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GC scan default details", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "GC", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 428, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 429, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(waiter_manager.*|deadlock_detect.*)\"}\n [$__rate_interval]\n)) by (instance, name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{name}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(waiter_manager.*|deadlock_detect.*)\"}\n [$__rate_interval]\n)) by (instance, name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock Manager Thread CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 430, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(waiter_manager.*|deadlock_detect.*)\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{name}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(waiter_manager.*|deadlock_detect.*)\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock Manager Thread CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 431, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_task_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_lock_manager_task_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock Manager Handled tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 432, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_lock_manager_waiter_lifetime_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Waiter lifetime duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 433, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(max_over_time(\n tikv_lock_manager_wait_table_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30s]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(max_over_time(\n tikv_lock_manager_wait_table_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30s]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(max_over_time(\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30s]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(max_over_time(\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30s]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock Waiting Queue", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 434, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_lock_manager_detect_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_lock_manager_detect_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_lock_manager_detect_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_lock_manager_detect_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Deadlock detect duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 435, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_manager_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_lock_manager_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Detect error", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 436, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(max_over_time(\n tikv_lock_manager_detector_leader_heartbeat\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30s]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(max_over_time(\n tikv_lock_manager_detector_leader_heartbeat\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30s]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Deadlock detector leader", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 437, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_pessimistic_lock_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_pessimistic_lock_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Total pessimistic locks memory size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 438, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_in_memory_pessimistic_locking\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{result}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_in_memory_pessimistic_locking\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "In-memory pessimistic locking result", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of active keys and waiters.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 439, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_lock_wait_queue_entries_gauge_vec\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Pessimistic lock activities", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Distribution of holder counts observed when writing non-empty shared-lock wrappers. Each observation represents one wrapper write, not the current live wrapper population.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 440, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_shared_lock_holder_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_shared_lock_holder_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Shared lock holder count", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate of shared-lock upgrade conflicts by reason.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 441, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_lock_upgrade_conflict_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (reason, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{reason}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_lock_upgrade_conflict_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (reason, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Shared lock upgrade conflicts", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The length includes the entering transaction itself", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 442, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_lock_wait_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_lock_wait_queue_length_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Lengths of lock wait queues when transaction enqueues", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration scan in-memory pessimistic locks with read lock", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 443, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_storage_mvcc_scan_lock_read_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "In-memory scan lock read duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Pessimistic Locking", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 444, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "total number of prewrite commands rejected by the lock limit", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 445, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_txn_file_prewrite_reject_by_lock_limit_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total", "metric": "", "query": "sum(rate(\n tikv_scheduler_txn_file_prewrite_reject_by_lock_limit_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_scheduler_txn_file_prewrite_reject_by_lock_limit_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_scheduler_txn_file_prewrite_reject_by_lock_limit_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scheduler reject prewrite", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "total number of denied splits due to locks or regions", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 446, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_rfstore_txn_file_split_denied_locks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (phase) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "locks - {{phase}}", "metric": "", "query": "sum(rate(\n tikv_rfstore_txn_file_split_denied_locks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (phase) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_rfstore_txn_file_split_denied_regions_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (phase) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "regions - {{phase}}", "metric": "", "query": "sum(rate(\n tikv_rfstore_txn_file_split_denied_regions_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (phase) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Denied split", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "total number of active resolve locks by result", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 447, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_rfstore_txn_file_active_resolve_locks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{result}}", "metric": "", "query": "sum(rate(\n tikv_rfstore_txn_file_active_resolve_locks_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Resolve locks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "duration of active resolve lock tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 448, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 2, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_rfstore_txn_file_active_resolve_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_rfstore_txn_file_active_resolve_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.5,(\n sum(rate(\n tikv_rfstore_txn_file_active_resolve_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "50%", "metric": "", "query": "histogram_quantile(0.5,(\n sum(rate(\n tikv_rfstore_txn_file_active_resolve_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Resolve locks duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "active resolve lock running tasks, schedule results, and concurrency limit", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 449, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_rfstore_txn_file_active_resolve_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "running tasks", "metric": "", "query": "sum((\n tikv_rfstore_txn_file_active_resolve_running_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_rfstore_txn_file_active_resolve_schedule_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "schedule - {{result}}", "metric": "", "query": "sum(rate(\n tikv_rfstore_txn_file_active_resolve_schedule_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_rfstore_txn_file_active_resolve_concurrency_limit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "concurrency limit", "metric": "", "query": "avg((\n tikv_rfstore_txn_file_active_resolve_concurrency_limit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Resolve locks tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Txn File", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 450, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of tasks handled by worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 451, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_worker_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Worker handled tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current pending and running tasks of worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 452, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_worker_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Worker pending tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of tasks handled by future_pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 453, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_futurepool_handled_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FuturePool handled tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current pending and running tasks of future_pool", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 454, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "sum(avg_over_time(\n tikv_futurepool_pending_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FuturePool pending tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Task", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 455, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of requests that TiKV sends to PD", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 456, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed by requests that TiKV sends to PD", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 457, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_pd_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_pd_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD request duration (average)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of PD heartbeat messages", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 458, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_pd_heartbeat_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_pd_pending_heartbeat_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-pending", "metric": "", "query": "sum((\n tikv_pd_pending_heartbeat_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD heartbeats", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total number of peers validated by the PD worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 459, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_pd_validate_peer_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_pd_validate_peer_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD validate peers", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of reconnection between TiKV and PD", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 460, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_pd_reconnect_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(delta(\n tikv_pd_reconnect_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD reconnection", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The forward status of PD client", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 461, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_pd_request_forwarded\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{host}}", "metric": "", "query": "((\n tikv_pd_request_forwarded\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PD forward status", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of TSO requests waiting in the queue.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 462, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_pd_pending_tso_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_pd_pending_tso_request_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Pending TSO Requests", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "PD", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 463, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration that recorded by inspecting messages.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 464, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n store_inspect_disk_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "disk-{{instance}}-{{type}}-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n store_inspect_disk_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n store_inspect_network_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, target, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "network-{{instance}}-{{target}}-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n store_inspect_network_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, target, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Inspected duration per server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The slow score of stores", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 465, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n store_slow_score\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n store_slow_score\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store Slow Score", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Slow Node Statistics", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 466, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The rate of Raft snapshot messages sent", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 467, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [1m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_raft_sent_message_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [1m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Rate snapshot message", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of snapshots in different states", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 468, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_raftstore_snapshot_traffic_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_raftstore_snapshot_traffic_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot state count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time snapshot generation/apply tasks spent waiting to be executed.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 469, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-generate-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_generation_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_apply_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-apply-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_apply_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot generation/apply wait duration $optional_quantile", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when handling snapshots", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 470, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_send_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "send-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_server_send_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "apply-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"generate\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "generate-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_raftstore_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"generate\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% Handle snapshot duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The snapshot size (P99.99).9999", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 471, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "size {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% Snapshot size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of KV within a snapshot in .9999", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 472, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_kv_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_snapshot_kv_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% Snapshot KV count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Action stats for snapshot generating and applying", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 473, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_snapshot_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, status, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{status}} {{$additional_groupby}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_snapshot_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, status, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_clean_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, status, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "clean-region-by-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(delta(\n tikv_raftstore_clean_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, status, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_server_snapshot_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(delta(\n tikv_server_snapshot_task_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot Actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The speed of sending or receiving snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 474, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_snapshot_limit_transport_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_snapshot_limit_transport_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_snapshot_limit_generate_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-generate-{{type}}", "metric": "", "query": "sum(rate(\n tikv_snapshot_limit_generate_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot transport speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of snapshots waiting to be applied", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 475, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_raftstore_snapshot_pending_applies\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tikv_raftstore_snapshot_pending_applies\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot pending applies", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Snapshot", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 476, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of resolved ts worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 477, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"resolved_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"resolved_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Resolved TS Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of resolved ts worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 478, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"resolved_ts.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"resolved_ts.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Resolved TS Worker CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of advance ts worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 479, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"advance_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"advance_ts.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Advance ts Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of advance ts worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 480, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"advance_ts.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"advance_ts.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Advance ts Worker CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of scan lock worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 481, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"inc_scan.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"inc_scan.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scan lock Worker CPU", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of scan lock worker", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 482, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"inc_scan.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"inc_scan.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Scan lock Worker CPU (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The gap between resolved ts (the maximum candidate of safe-ts) and current time.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 483, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max gap of resolved-ts", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The region that has minimal resolved ts", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 484, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Min Resolved TS Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The gap between now() and the minimal (non-zero) safe ts for followers", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 485, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_follower_safe_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_follower_safe_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max gap of follower safe-ts", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The region id of the follower that has minimal safe ts", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 486, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_follower_safe_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_follower_safe_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Min Safe TS Follower Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The gap between resolved ts of leaders and current time", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 487, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_gap_millis\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max gap of resolved-ts in region leaders", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The region that its leader has minimal resolved ts.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 488, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_resolved_ts_min_leader_resolved_ts_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Min Leader Resolved TS Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when handle a check leader request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 489, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_resolved_ts_check_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_resolved_ts_check_leader_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Check leader duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Bucketed histogram of region count in a check leader request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 490, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CheckLeader request region count $optional_quantile", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Bucketed histogram of the check leader request size", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 491, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_check_leader_request_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_check_leader_request_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-check-num-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_check_leader_request_item_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CheckLeader request size$optional_quantile", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of fail to advance resolved-ts", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 492, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_resolved_ts_fail_advance_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{reason}}", "metric": "", "query": "sum(delta(\n tikv_resolved_ts_fail_advance_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_raftstore_check_stale_peer\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-stale-peer", "metric": "", "query": "sum(delta(\n tikv_raftstore_check_stale_peer\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Fail advance ts count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total bytes in memory of resolved-ts observe regions's lock heap", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 493, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_resolved_ts_lock_heap_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "avg((\n tikv_resolved_ts_lock_heap_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Lock heap size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The backoff duration before starting initial scan", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 494, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_resolved_ts_initial_scan_backoff_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Initial scan backoff duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The status of resolved-ts observe regions", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 495, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resolved_ts_region_resolve_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_resolved_ts_region_resolve_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Observe region status", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total bytes of pending commands in the channel", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 496, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_resolved_ts_channel_penging_cmd_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "avg((\n tikv_resolved_ts_channel_penging_cmd_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Pending command size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Resolved TS", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 497, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 498, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 499, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Usage (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 500, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"queue|exec_download\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "(DL){{type}}-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"queue|exec_download\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "(AP){{type}}-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "P99 RPC Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 501, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n [$__rate_interval]\n)) by (instance, request) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{request}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n [$__rate_interval]\n)) by (instance, request) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-{{request}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Ops", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 502, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 503, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_apply_cache_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_apply_cache_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cache Events", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 504, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Overall RPC Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 505, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"exec_download\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_apply_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"exec_download\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Read File into Memory Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 506, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"queuing\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"queuing\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Queuing Time", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 507, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_apply_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_apply_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply Request Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 508, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_download_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_download_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Downloaded File Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 509, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_apply_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_apply_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply Batch Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 510, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"get_permit\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"get_permit\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Blocked by Concurrency Time", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 511, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply Request Speed", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 512, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_import_apply_cached_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_import_apply_cached_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cached File in Memory", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 513, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_import_applier_event\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type!=\"begin_req\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Engine Requests Unfinished", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 514, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_engine_request_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"apply\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply Time", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 515, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raftstore-.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_server_mem_trace_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"raftstore-.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Store Memory Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Point In Time Restore", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 516, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 517, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(b.*k.*w.*k.*|backup_io)\"}\n [$__rate_interval]\n)) by (instance, name) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{name}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(b.*k.*w.*k.*|backup_io)\"}\n [$__rate_interval]\n)) by (instance, name) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup CPU Utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 518, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(b.*k.*w.*k.*|backup_io)\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{name}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(b.*k.*w.*k.*|backup_io)\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup CPU Utilization (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 519, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_backup_thread_pool_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_backup_thread_pool_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup Thread Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 520, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_backup_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, error) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{error}}", "metric": "", "query": "sum(delta(\n tikv_backup_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, error) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 521, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Write CF SST Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 14 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 522, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"default\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_backup_range_size_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",cf=\"default\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Default CF SST Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 523, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total", "metric": "", "query": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{cf}}", "metric": "", "query": "sum(rate(\n tikv_backup_range_size_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, cf) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup SST Generation Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 524, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"snapshot\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Scan SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 525, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"scan\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"scan\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Scan SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 21 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 526, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"save.*\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"save.*\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Backup Save SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 527, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-99.9% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-$optional_quantile {{$additional_groupby}}", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_backup_range_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum((\n tikv_backup_range_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) / sum((\n tikv_backup_range_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-avg {{$additional_groupby}}", "metric": "", "query": "(sum((\n tikv_backup_range_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) / sum((\n tikv_backup_range_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Backup SST Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 528, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "External Storage Create Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 529, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_external_storage_create_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_external_storage_create_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_external_storage_create_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_external_storage_create_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "External Storage Create Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 530, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{req}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{req}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) / sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{req}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",req=~\"analyze.*|checksum.*\"}\n [$__rate_interval]\n)) by (req, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Checksum Request Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 531, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, device) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{device}}", "metric": "", "query": "sum(rate(\n node_disk_io_time_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, device) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IO Utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 532, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, tid, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{tid}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, tid, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import CPU Utilization", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 533, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, tid, kind, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{tid}} {{kind}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance, tid, kind, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import CPU Utilization (by kind)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 534, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "count(rate(\n tikv_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"sst_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Thread Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 535, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_import_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, error, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{error}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_import_error_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, error, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 536, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{request}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{request}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_import_rpc_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) / sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{request}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_import_rpc_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) / sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{request}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 537, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{request}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_import_rpc_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request!=\"switch_mode\"}\n [$__rate_interval]\n)) by (request, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Ops", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 538, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "((\n tikv_import_rpc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import RPC Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 63 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 539, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=~\"download|write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=~\"download|write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Write/Download RPC Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 63 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 540, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"queue\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"queue\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Wait Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 63 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 541, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"read\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"read\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Read SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 63 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 542, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"rewrite\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_download_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"rewrite\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Rewrite SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 70 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 543, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_rpc_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",request=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Ingest RPC Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 70 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 544, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_ingest_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_ingest_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Ingest SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 70 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 545, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_import_ingest_byte_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_import_ingest_byte_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Import Ingest SST Bytes", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 546, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total", "metric": "", "query": "sum(rate(\n tikv_import_download_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Download SST Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 547, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_import_local_write_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_import_local_write_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Local Write keys", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 548, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_import_local_write_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tikv_import_local_write_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Import Local Write bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 549, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "sum", "metric": "", "query": "sum((\n tikv_backup_raw_expired_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL Expired", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 550, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_cloud_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (cloud, req) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cloud}}-{{req}}", "metric": "", "query": "sum(rate(\n tikv_cloud_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (cloud, req) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "cloud request", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Backup & Import", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 551, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": null, "mappings": [ { "options": { "0": { "color": "red", "index": null, "text": "Disabled" }, "1": { "color": "green", "index": null, "text": "Enabled" } }, "type": "value" } ], "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 552, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tikv_log_backup_enabled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "((\n tikv_log_backup_enabled\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Endpoint Status", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": null, "mappings": [ { "options": { "0": { "color": "green", "index": null, "text": "Running" }, "1": { "color": "yellow", "index": null, "text": "Paused" }, "2": { "color": "red", "index": null, "text": "Error" } }, "type": "value" } ], "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 553, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "min((\n tikv_log_backup_task_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "min((\n tikv_log_backup_task_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Task Status", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 554, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "name" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "tidb_log_backup_advancer_owner > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "tidb_log_backup_advancer_owner > 0", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Advancer Owner", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The average flush size of last 30mins.", "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 555, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(increase(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) / sum(increase(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n [30m]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "(sum(increase(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) / sum(increase(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n [30m]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Average Flush Size", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The current total flushed file number of this run.", "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": 0, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 556, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) > 0", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flushed Files (Last 30m) Per Host", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": 0, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 557, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n [30m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n [30m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flush Times (Last 30m)", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "This is the summary of the size has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 558, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_file_size_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Total Flushed Size (Last 30m)", "transformations": [], "transparent": false, "type": "stat" }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "This is the summary of the file count has been flushed, summered by the data each TiKV has flushed since last boot.\n**NOTE: The size may get reduced if some of TiKVs reboot.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": 0, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 559, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_flush_file_size_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [30m]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flush Files (Last 30m)", "transformations": [], "transparent": false, "type": "stat" }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The CPU utilization of log backup threads. \n**(Note this is the average usage for a period of time, some peak of CPU usage may be lost.)**", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 560, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/ user$/", "color": "#2ca02c" }, { "alias": "/ sys$/", "color": "#f2c12e" }, { "alias": "/ run_queue_wait$/", "color": "#ff8c00" }, { "alias": "/ blkio_wait$/", "color": "#d62728" } ], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(backup_stream|log-backup-scan(-[0-9]+)?)\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} {{name}} {{kind}}", "metric": "", "query": "sum(rate(\n tikv_thread_cpu_detail_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",name=~\"(backup_stream|log-backup-scan(-[0-9]+)?)\"}\n [$__rate_interval]\n)) by (instance, name, kind) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 561, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_handle_kv_batch_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_handle_kv_batch_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Handle Event Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The data rate of initial scanning emitting events.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 562, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_incremental_scan_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_incremental_scan_bytes_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scan Generate Event Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 563, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(time() * 1000 - max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (task) / 262144 > 0)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ task }}", "metric": "", "query": "(time() * 1000 - max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (task) / 262144 > 0)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Abnormal Checkpoint TS Lag", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The estimated memory usage by the streaming backup module.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 564, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_log_backup_heap_memory\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_log_backup_heap_memory\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Of Events", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 565, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-total", "metric": "", "query": "sum((\n tikv_log_backup_observed_region\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Observed Region Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The errors met when backing up.\n**They are retryable, don't worry.**", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 566, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The errors met when backing up.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 567, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tikv_log_backup_fatal_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(delta(\n tikv_log_backup_fatal_errors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [1m]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Fatal Errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 568, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Current Time", "bars": false, "dashes": true, "fill": 0, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (task) / 262144 > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{task}}", "metric": "", "query": "max((\n tidb_log_backup_last_checkpoint\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (task) / 262144 > 0", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "time() * 1000", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Current Time", "metric": "", "query": "time() * 1000", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Checkpoint TS of Tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "dateTimeAsIsoNoDateIfToday", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of flushing a batch of file.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 569, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_flush_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_flush_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_files\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Flush Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of scanning the initial data from local DB and transform them into apply events.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 570, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_initial_scan_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_initial_scan_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Initial scanning duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of converting a raft request into a apply event.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 571, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"to_stream_event\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"to_stream_event\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Convert Raft Event duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of waiting the mutex of the controller.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 42 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 572, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"get_router_lock\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"get_router_lock\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Wait for Lock Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The number of KV-modify of each raft command observed.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 573, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_handle_kv_batch_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_handle_kv_batch_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Command Batch Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The total cost of saving an event into temporary file.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 574, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_to_temp_file\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_event_handle_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=~\"save_to_temp_file\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Save to Temp File Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The total cost of writing a event into temporary file.\nComparing to the ***Save*** duration, it doesn't contain the time cost of routing the task by range / task.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 575, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=\"write_to_tempfile\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=\"write_to_tempfile\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Write to Temp File Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "The duration of collecting metadata and call the UNIX system call *write* for each event.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 49 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 576, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=\"syscall_write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tikv_log_backup_on_event_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",stage=\"syscall_write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "System Write Call Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal message type count.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 577, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (message) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{message}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (message) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Internal Message Type", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 2, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 578, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{message}}-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Internal Message Handling Duration $optional_quantile", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 579, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{message}}", "metric": "", "query": "histogram_quantile(0.9,(\n sum(rate(\n tikv_log_backup_internal_actor_acting_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (message, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Internal Message Handling Duration (P90)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal read throughput of RocksDB during initial scanning. This panel can roughly present the read through to the hard disk of initial scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 580, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op=~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scan RocksDB Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Misc statistics of RocksDB during initial scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 581, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op!~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf, op) > 0", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{cf}}-{{op}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_initial_scan_operations\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",op!~\"read_bytes\"}\n [$__rate_interval]\n)) by (cf, op) > 0", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scan RocksDB Operation", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The reason of triggering initial scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 582, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_initial_scan_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{reason}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_initial_scan_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scanning Trigger Reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The task status of initial scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 583, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_log_backup_pending_initial_scan\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (stage) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{stage}}", "metric": "", "query": "sum((\n tikv_log_backup_pending_initial_scan\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (stage) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Initial Scanning Task Status", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 584, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_log_backup_metadata_key_operation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tikv_log_backup_metadata_key_operation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Checkpoint Key Putting", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 585, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tidb_log_backup_advancer_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"checkpoint\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tidb_log_backup_advancer_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"checkpoint\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Request Checkpoint Batch Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 586, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"tick\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{le}}", "metric": "", "query": "sum(increase(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"tick\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Tick Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The reason of advancer failed to be advanced.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 587, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_region_request_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",reason!=\"retryable-scan-region\"}\n [$__rate_interval]\n)) by (reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{reason}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_region_request_failure\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",reason!=\"retryable-scan-region\"}\n [$__rate_interval]\n)) by (reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Checkpoint Failure Reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The result of getting region checkpoints.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 588, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_region_request\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{result}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_region_request\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (result) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request Result", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 589, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile($optional_quantile,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{step}}-$optional_quantile", "metric": "", "query": "histogram_quantile($optional_quantile,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Tick Duration $optional_quantile", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The internal handling message duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 590, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ step }}", "metric": "", "query": "histogram_quantile(0.9,(\n sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (step, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Tick Duration (P90)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The frequent of getting region level checkpoint.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 591, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"get-regions-in-range\"}\n [$__rate_interval]\n)) by (step, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{step}}-{{instance}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"get-regions-in-range\"}\n [$__rate_interval]\n)) by (step, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Get Region Operation Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The variant of checkpoint group.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 98 }, "height": null, "hideTimeOverride": false, "id": 592, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"try-advance\"}\n [$__rate_interval]\n)) by (step, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{step}}-{{instance}}", "metric": "", "query": "sum(rate(\n tidb_log_backup_advancer_tick_duration_sec_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",step=\"try-advance\"}\n [$__rate_interval]\n)) by (step, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Try Advance Trigger Time", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Backup Log", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 593, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 594, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, state) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{state}}", "metric": "", "query": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, state) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-total", "metric": "", "query": "sum((\n tikv_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads state", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 595, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n sum(rate(\n tikv_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, io) > 1024\n \n \n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}} {{$additional_groupby}}", "metric": "", "query": "topk(20,(\n sum(rate(\n tikv_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name, io) > 1024\n \n \n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads IO", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 596, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n max(rate(\n tikv_thread_voluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "topk(20,(\n max(rate(\n tikv_thread_voluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Thread Voluntary Context Switches", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 597, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "topk(20,(\n max(rate(\n tikv_thread_nonvoluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}", "metric": "", "query": "topk(20,(\n max(rate(\n tikv_thread_nonvoluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (name) > 100\n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Thread Nonvoluntary Context Switches", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Threads", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 598, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 599, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_allocator_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n tikv_allocator_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Allocator Stats", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 600, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) - sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{thread_name}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) - sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Send Allocated(+) / Release Received(-) Bytes Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 601, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{thread_name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"alloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Allocated Bytes Rate per Thread", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 602, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{thread_name}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_allocator_thread_allocation\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"dealloc\"}\n [$__rate_interval]\n)) by (thread_name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Released Bytes Rate per Thread", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 603, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_allocator_thread_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"mapped\"}\n \n)) by (thread_name, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{thread_name}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tikv_allocator_thread_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"mapped\"}\n \n)) by (thread_name, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Mapped Allocation per Thread", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 604, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_allocator_arena_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_allocator_arena_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Arena Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Auto dump action rates grouped by component and action", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 605, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_memory_profile_auto_dump_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n [$__rate_interval]\n)) by (component, action) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{component}}-{{action}}", "metric": "", "query": "sum(rate(\n tikv_memory_profile_auto_dump_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",job=~\".*(tikv|copr|schema-manager).*\"}\n [$__rate_interval]\n)) by (component, action) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Profile Auto Dump Actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Memory", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 606, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 607, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_resource_control_background_task_wait_duration\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{resource_group}}", "metric": "", "query": "sum(rate(\n tikv_resource_control_background_task_wait_duration\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Background Task Total Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memory usage of the resource control module.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 608, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_resource_control_priority_quota_limit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, priority) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{priority}}", "metric": "", "query": "sum((\n tikv_resource_control_priority_quota_limit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance, priority) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Priority Quota Limit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of CPU throttle token allocations by result", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 609, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_cpu_throttle_allocations_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, result) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{ result }}", "metric": "", "query": "sum(rate(\n tikv_cpu_throttle_allocations_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, result) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_cpu_throttle_low_priority_dag_allocations_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, result) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-low-pri-{{ result }}", "metric": "", "query": "sum(rate(\n tikv_cpu_throttle_low_priority_dag_allocations_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, result) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Throttle Token Allocation Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Global CPU usage ratio (0.0 - 1.0)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 610, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_cpu_usage_monitor_global_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "avg((\n tikv_cpu_usage_monitor_global_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Global CPU Usage Ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Per-keyspace CPU usage ratio (0.0 - 1.0)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 611, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_cpu_usage_monitor_keyspace_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "avg((\n tikv_cpu_usage_monitor_keyspace_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Per-Keyspace CPU Usage Ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of refill rate adjustments (dynamic adjustment)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 612, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_cpu_throttle_refill_rate_adjustments_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, level, direction) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{ level }}-{{ direction }}", "metric": "", "query": "sum(rate(\n tikv_cpu_throttle_refill_rate_adjustments_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, level, direction) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Throttle Refill Rate Adjustments", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of burst allocations per keyspace", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 613, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_cpu_throttle_burst_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "sum(rate(\n tikv_cpu_throttle_burst_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Throttle Burst Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of CPU tokens used in burst mode (microseconds)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 614, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_cpu_throttle_burst_tokens_used_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "sum(rate(\n tikv_cpu_throttle_burst_tokens_used_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Throttle Burst Tokens Used", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of fair allocation triggers when global CPU > threshold", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 615, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_cpu_throttle_fair_allocations_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "sum(rate(\n tikv_cpu_throttle_fair_allocations_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Throttle Fair Allocations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of retries during CPU token allocation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 28 }, "height": null, "hideTimeOverride": false, "id": 616, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_cpu_token_allocation_retries_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "sum(rate(\n tikv_cpu_token_allocation_retries_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Token Allocation Retries", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current uniform per-tenant CPU cap ratio (0 = disabled)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 617, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_cpu_throttle_tenant_cap_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "max((\n tikv_cpu_throttle_tenant_cap_ratio\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Tenant CPU Cap Ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Computed uniform per-tenant CPU cap quota (microseconds per refill interval; 0 = disabled)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 618, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_cpu_throttle_tenant_cap_quota_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "max((\n tikv_cpu_throttle_tenant_cap_quota_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Tenant CPU Cap Quota", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of allocations rejected because the per-tenant CPU cap was exhausted with no burst available", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 35 }, "height": null, "hideTimeOverride": false, "id": 619, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_cpu_throttle_allocations_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",result=\"tenant_cap_exhausted\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "sum(rate(\n tikv_cpu_throttle_allocations_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",result=\"tenant_cap_exhausted\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Tenant CPU Cap Exhausted", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Average wait duration for CPU token allocation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 620, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_cpu_token_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result) / sum(rate(\n tikv_cpu_token_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}-{{ result }}", "metric": "", "query": "(sum(rate(\n tikv_cpu_token_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result) / sum(rate(\n tikv_cpu_token_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_cpu_throttle_low_priority_dag_cpu_token_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, result) / sum(rate(\n tikv_cpu_throttle_low_priority_dag_cpu_token_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, result) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "low-pri-{{ instance }}-{{ result }}", "metric": "", "query": "(sum(rate(\n tikv_cpu_throttle_low_priority_dag_cpu_token_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, result) / sum(rate(\n tikv_cpu_throttle_low_priority_dag_cpu_token_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, result) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Token Wait Duration (Avg)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "P99 wait duration for CPU token allocation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 621, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_token_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}-{{ result }}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_token_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_low_priority_dag_cpu_token_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, result, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "low-pri-{{ instance }}-{{ result }}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_low_priority_dag_cpu_token_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, result, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Token Wait Duration (P99)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Success-path low-priority DAG total wait duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 49 }, "height": null, "hideTimeOverride": false, "id": 622, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_cpu_throttle_low_priority_dag_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",result=\"success\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_cpu_throttle_low_priority_dag_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",result=\"success\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-avg-{{ instance }}", "metric": "", "query": "(sum(rate(\n tikv_cpu_throttle_low_priority_dag_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",result=\"success\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_cpu_throttle_low_priority_dag_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",result=\"success\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_low_priority_dag_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",result=\"success\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-p99-{{ instance }}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_low_priority_dag_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",result=\"success\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Low-Priority DAG Wait Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Available tokens / capacity for global bucket", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 623, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(avg((\n tikv_cpu_throttle_global_bucket_available_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) / avg((\n tikv_cpu_throttle_global_bucket_capacity_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "(avg((\n tikv_cpu_throttle_global_bucket_available_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) / avg((\n tikv_cpu_throttle_global_bucket_capacity_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(avg((\n tikv_cpu_throttle_low_priority_dag_bucket_available_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) / avg((\n tikv_cpu_throttle_low_priority_dag_bucket_capacity_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "low-pri-{{ instance }}", "metric": "", "query": "(avg((\n tikv_cpu_throttle_low_priority_dag_bucket_available_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) / avg((\n tikv_cpu_throttle_low_priority_dag_bucket_capacity_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Global Token Bucket Availability Ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Available tokens / capacity for keyspace buckets", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 624, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(avg((\n tikv_cpu_throttle_keyspace_bucket_available_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) / avg((\n tikv_cpu_throttle_keyspace_bucket_capacity_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "(avg((\n tikv_cpu_throttle_keyspace_bucket_available_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) / avg((\n tikv_cpu_throttle_keyspace_bucket_capacity_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keyspace Token Bucket Availability Ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current global refill rate (us per interval)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 625, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_cpu_throttle_global_refill_rate_us_per_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}", "metric": "", "query": "avg((\n tikv_cpu_throttle_global_refill_rate_us_per_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_cpu_throttle_low_priority_dag_refill_rate_us_per_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "low-pri-{{ instance }}", "metric": "", "query": "avg((\n tikv_cpu_throttle_low_priority_dag_refill_rate_us_per_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Global Token Bucket Refill Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current keyspace refill rate (us per interval)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 63 }, "height": null, "hideTimeOverride": false, "id": 626, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_cpu_throttle_keyspace_refill_rate_us_per_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "avg((\n tikv_cpu_throttle_keyspace_refill_rate_us_per_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keyspace Token Bucket Refill Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Average wait duration for runtime token allocation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 627, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_cpu_token_runtime_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result) / sum(rate(\n tikv_cpu_token_runtime_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}-{{ result }}", "metric": "", "query": "(sum(rate(\n tikv_cpu_token_runtime_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result) / sum(rate(\n tikv_cpu_token_runtime_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Runtime Token Wait Duration (Avg)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "P99 wait duration for runtime token allocation", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 70 }, "height": null, "hideTimeOverride": false, "id": 628, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_token_runtime_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}-{{ result }}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_token_runtime_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, result, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Runtime Token Wait Duration (P99)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Average CPU time per throttled request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 629, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_cpu_throttle_request_cpu_time_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) / sum(rate(\n tikv_cpu_throttle_request_cpu_time_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "(sum(rate(\n tikv_cpu_throttle_request_cpu_time_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) / sum(rate(\n tikv_cpu_throttle_request_cpu_time_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_cpu_throttle_low_priority_dag_request_cpu_time_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_cpu_throttle_low_priority_dag_request_cpu_time_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "low-pri-{{ instance }}", "metric": "", "query": "(sum(rate(\n tikv_cpu_throttle_low_priority_dag_request_cpu_time_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_cpu_throttle_low_priority_dag_request_cpu_time_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Throttled Request CPU Time (Avg)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "P99 CPU time per throttled request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 77 }, "height": null, "hideTimeOverride": false, "id": 630, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_request_cpu_time_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_request_cpu_time_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_low_priority_dag_request_cpu_time_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "low-pri-{{ instance }}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_low_priority_dag_request_cpu_time_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Throttled Request CPU Time (P99)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Average ratio of actual CPU time to estimated", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 631, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_cpu_throttle_request_actual_to_estimated_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) / sum(rate(\n tikv_cpu_throttle_request_actual_to_estimated_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "(sum(rate(\n tikv_cpu_throttle_request_actual_to_estimated_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) / sum(rate(\n tikv_cpu_throttle_request_actual_to_estimated_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_cpu_throttle_low_priority_dag_actual_to_estimated_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_cpu_throttle_low_priority_dag_actual_to_estimated_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "low-pri-{{ instance }}", "metric": "", "query": "(sum(rate(\n tikv_cpu_throttle_low_priority_dag_actual_to_estimated_ratio_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_cpu_throttle_low_priority_dag_actual_to_estimated_ratio_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Actual/Estimated CPU Ratio (Avg)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "P99 ratio of actual CPU time to estimated", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 84 }, "height": null, "hideTimeOverride": false, "id": 632, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_request_actual_to_estimated_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_request_actual_to_estimated_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n [$__rate_interval]\n)) by (instance, keyspace_id, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_low_priority_dag_actual_to_estimated_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "low-pri-{{ instance }}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_cpu_throttle_low_priority_dag_actual_to_estimated_ratio_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Actual/Estimated CPU Ratio (P99)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current keyspace service limit value from PD (RU/s)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 633, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_service_limit_keyspace_value\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "avg((\n tikv_service_limit_keyspace_value\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Service Limit Keyspace Value", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "CPU quota derived from PD service limit (us per interval)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 91 }, "height": null, "hideTimeOverride": false, "id": 634, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tikv_service_limit_keyspace_quota_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{ instance }}-KS{{ keyspace_id }}", "metric": "", "query": "avg((\n tikv_service_limit_keyspace_quota_us\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",keyspace_id=~\"$keyspace_id\"}\n \n)) by (instance, keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Service Limit Keyspace Quota", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Resource Control", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 635, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "count of different kinds of request wait", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 636, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum(rate(\n tikv_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request wait count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "total bytes of read in each active keyspace", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 637, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_active_keyspace_read_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{keyspace_id}}-read", "metric": "", "query": "sum(rate(\n tikv_active_keyspace_read_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, keyspace_id) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keyspace read MBps", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "wait time of request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 638, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_request_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) / sum(rate(\n tikv_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "(sum(rate(\n tikv_request_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) / sum(rate(\n tikv_request_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Average request wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "wait time of request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 639, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99% request wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "wait time of request", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 640, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_request_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.99% request wait duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Resource Control (Serverless)", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 641, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The 99 quantile durtion of status server API requests", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 642, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{path}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{path}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_status_server_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_status_server_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) / sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{path}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_status_server_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) / sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{path}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Status API Request Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 643, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{path}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_status_server_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Status API Request (op/s)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Status Server", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 644, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of regions with encryption enabled", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 645, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_encryption_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_encryption_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encrypted region count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Number of keyspaces with encryption enabled", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 646, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_encryption_keyspace_active_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tikv_encryption_keyspace_active_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encrypted keyspace count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.0, "yaxis": "left" } ] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 647, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_encryption_key_unavailable_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "key-unavailable-{{instance}}-{{reason}}", "metric": "", "query": "sum(rate(\n tikv_encryption_key_unavailable_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance, reason) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_encryption_keyspace_cmek_health_check_failure_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "health-check-failed-{{instance}}", "metric": "", "query": "sum((\n tikv_encryption_keyspace_cmek_health_check_failure_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tikv_rfstore_encryption_apply_paused_peers\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "cmek-apply-paused-{{instance}}", "metric": "", "query": "sum((\n tikv_rfstore_encryption_apply_paused_peers\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [ { "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.0, "yaxis": "left" } ], "timeFrom": null, "timeShift": null, "title": "Encryption errors", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time spent encrypting or decrypting raft log entries in the write path.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 648, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n rfstore_raft_entry_crypt_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n rfstore_raft_entry_crypt_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n rfstore_raft_entry_crypt_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n rfstore_raft_entry_crypt_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n rfstore_raft_entry_crypt_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n rfstore_raft_entry_crypt_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n rfstore_raft_entry_crypt_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n rfstore_raft_entry_crypt_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n rfstore_raft_entry_crypt_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n rfstore_raft_entry_crypt_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft log encrypt/decrypt duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Time spent encrypting or decrypting SST data in the KV engine.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 649, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_sst_crypt_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n kv_engine_sst_crypt_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_sst_crypt_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n kv_engine_sst_crypt_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n kv_engine_sst_crypt_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n kv_engine_sst_crypt_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n kv_engine_sst_crypt_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n kv_engine_sst_crypt_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n kv_engine_sst_crypt_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n kv_engine_sst_crypt_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "KV engine SST encrypt/decrypt duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Master key generating or decrypting duration, including KMS round trips.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 650, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_master_key_ops_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_master_key_ops_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_master_key_ops_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99%-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_master_key_ops_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_master_key_ops_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_master_key_ops_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_master_key_ops_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tikv_master_key_ops_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_master_key_ops_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_master_key_ops_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Master Key Generate/Decrypt duration (calls KMS)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Encryption Key read duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 651, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_encryption_key_get_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_encryption_key_get_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_encryption_key_get_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_encryption_key_get_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_encryption_key_get_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_encryption_key_get_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_encryption_key_get_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_encryption_key_get_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_encryption_key_get_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_encryption_key_get_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encryption Key Read Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Rate of TiKV switching to a previous data key to decrypt older data.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 652, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_encryption_key_switch_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_encryption_key_switch_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Encrypted key switch rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Encryption", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 653, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 654, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_expire_kv_count_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_ttl_expire_kv_count_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL expire count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 655, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_expire_kv_size_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tikv_ttl_expire_kv_size_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL expire size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 656, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_ttl_checker_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n [$__rate_interval]\n)) by (instance) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "(sum(rate(\n tikv_ttl_checker_processed_regions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (instance) / sum(rate(\n tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"region\"}\n [$__rate_interval]\n)) by (instance) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL check progress", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 7 }, "height": null, "hideTimeOverride": false, "id": 657, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_checker_actions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_ttl_checker_actions\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL checker actions", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The time consumed when executing GC tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 658, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^count/", "bars": false, "dashLength": 1, "dashes": true, "fill": 2, "fillBelowTo": null, "lines": true, "spaceLength": 1, "transform": "negative-Y", "yaxis": 2, "zindex": -3 }, { "alias": "/^avg/", "bars": false, "fill": 7, "fillBelowTo": null, "lines": true, "yaxis": 1, "zindex": 0 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99.99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99% {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tikv_ttl_checker_compact_duration_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tikv_ttl_checker_compact_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tikv_ttl_checker_compact_duration_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tikv_ttl_checker_compact_duration_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TTL checker compact duration", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "color": null, "custom": {}, "decimals": null, "mappings": null, "noValue": "none", "thresholds": { "mode": "absolute", "steps": "" }, "unit": "ms" }, "overrides": [] }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 14 }, "height": null, "hideTimeOverride": false, "id": 659, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "options": { "colorMode": "value", "graphMode": "none", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": [ "lastNotNull" ], "fields": "", "values": false }, "textMode": "auto" }, "repeat": null, "repeatDirection": null, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tikv_ttl_checker_poll_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"tikv_gc_run_interval\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": null, "metric": "", "query": "max((\n tikv_ttl_checker_poll_interval\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"${instance:regex}\",instance=~\"${worker_instance:regex}\",type=\"tikv_gc_run_interval\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "TTL checker poll interval", "transformations": [], "transparent": false, "type": "stat" } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "TTL", "transformations": [], "transparent": false, "type": "row" } ], "refresh": "1m", "rows": [], "schemaVersion": 14, "sharedCrosshair": false, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "k8s_cluster", "multi": false, "name": "k8s_cluster", "options": [], "query": "label_values(tikv_engine_size_bytes, k8s_cluster)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", "options": [], "query": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "db", "multi": true, "name": "db", "options": [], "query": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "command", "multi": true, "name": "command", "options": [], "query": "query_result(tikv_storage_command_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}!=0)", "refresh": 2, "regex": "/\\btype=\"([^\"]+)\"/", "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": ".*", "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "instance", "multi": true, "name": "instance", "options": [], "query": "label_values(tikv_engine_size_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": ".*", "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "worker_instance", "multi": true, "name": "worker_instance", "options": [], "query": "label_values(tikv_worker_cpu_cores_quota{k8s_cluster =\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": true, "label": "titan_db", "multi": true, "name": "titan_db", "options": [], "query": "label_values(tikv_engine_titandb_num_live_blob_file{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, db)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": {}, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": false, "label": "additional_groupby", "multi": false, "name": "additional_groupby", "options": [ { "selected": false, "text": "none", "value": "none" }, { "selected": false, "text": "instance", "value": "instance" } ], "query": "none,instance", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "custom", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": {}, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": false, "label": "optional_quantile", "multi": false, "name": "optional_quantile", "options": [ { "selected": false, "text": "0.99", "value": "0.99" }, { "selected": false, "text": "0.999", "value": "0.999" }, { "selected": false, "text": "0.9999", "value": "0.9999" }, { "selected": false, "text": "0.99999", "value": "0.99999" }, { "selected": false, "text": "1", "value": "1" } ], "query": "0.99,0.999,0.9999,0.99999,1", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "custom", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "keyspace_id", "multi": true, "name": "keyspace_id", "options": [], "query": "label_values({__name__=~\"tikv_cpu_token_wait_duration_seconds_count|tikv_yatp_pool_keyspace_task_slice_cpu_duration_seconds_count\", k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, keyspace_id)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "hidden": false, "nowDelay": null, "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Test-Cluster-TiKV-Details", "uid": "RDVQiEzZz", "version": 0 }