{ "__inputs": [ { "description": "", "label": "Test-Cluster", "name": "DS_TEST-CLUSTER", "pluginId": "prometheus", "pluginName": "Prometheus", "type": "datasource" } ], "annotations": { "list": [] }, "description": "", "editable": true, "gnetId": null, "graphTooltip": 1, "hideControls": false, "id": null, "links": [], "panels": [ { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 1, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 3, "description": "The storage size per TiFlash instance.\n(Not including some disk usage of TiFlash-Proxy by now)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 2, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 0, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_StoreSizeUsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-local", "metric": "", "query": "sum((\n tiflash_system_current_metric_StoreSizeUsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_StoreSizeUsedRemote\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-remote", "metric": "", "query": "sum((\n tiflash_system_current_metric_StoreSizeUsedRemote\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Store size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 3, "description": "The available capacity size per TiFlash instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 3, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 0, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_StoreSizeAvailable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_StoreSizeAvailable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Available size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 3, "description": "The capacity size per TiFlash instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 5, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 4, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 0, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": true, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_StoreSizeCapacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_StoreSizeCapacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Capacity size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "TiFlash uptime since last restart", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 5, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_asynchronous_metric_Uptime\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tiflash_system_asynchronous_metric_Uptime\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Uptime", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "dtdurations", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 0, "description": "The number of Regions on each TiFlash instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 6, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_proxy_tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",type=\"region\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tiflash_proxy_tikv_raftstore_region_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",type=\"region\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_proxy_tikv_raftstore_hibernated_peer_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, state) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{state}}", "metric": "", "query": "sum((\n tiflash_proxy_tikv_raftstore_hibernated_peer_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, state) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "TiFlash CPU usage calculated with process CPU running seconds.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 7, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/limit/", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=~\".*tiflash\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[$__rate_interval])", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "rate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=~\".*tiflash\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"}[$__rate_interval])", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_LogicalCPUCores\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "limit-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_LogicalCPUCores\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memory usage per TiFlash instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 8, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/limit/", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_proxy_process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tiflash_proxy_process_resident_memory_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_MemoryCapacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "limit-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_MemoryCapacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_jemalloc_retained\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "retained", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_jemalloc_retained\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_jemalloc_mapped\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mapped", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_jemalloc_mapped\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_jemalloc_resident\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "resident", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_jemalloc_resident\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_jemalloc_allocated\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "allocated", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_jemalloc_allocated\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_jemalloc_active\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "active", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_jemalloc_active\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_jemalloc_metadata_thp\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "metadata_thp", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_jemalloc_metadata_thp\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_jemalloc_metadata\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "metadata", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_jemalloc_metadata\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_mimalloc_current_rss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mimalloc_rss", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_mimalloc_current_rss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_mimalloc_current_commit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mimalloc_commit", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_mimalloc_current_commit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_mmap_alive\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mmap", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_mmap_alive\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 9, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(irate(\n tiflash_proxy_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(irate(\n tiflash_proxy_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IO Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 10, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_s3_store_summary_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, store_id, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "store-{{store_id}}-{{type}}", "metric": "", "query": "sum((\n tiflash_storage_s3_store_summary_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, store_id, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Store Summary (Disagg arch)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Server", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 11, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Involved when importing data.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 12, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"sst_importer.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"sst_importer.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"sst_importer.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"sst_importer.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "SST Import Service", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Involved when importing data.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 13, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"apply_low_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"apply_low_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"apply_low_.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"apply_low_.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "SST Apply", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 14, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"region_task.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"region_task.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"region_task.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"region_task.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Task", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 15, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"region_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"region_worker.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"region_worker.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"region_worker.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Region Worker", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 16, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"raftstore_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"raftstore_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"raftstore_.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"raftstore_.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Store", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 17, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"apply_.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"apply_.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"apply_.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"apply_.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply Worker", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 18, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"bg_\\\\d+\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"bg_\\\\d+\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"bg_\\\\d+\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"bg_\\\\d+\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage Background (Small Tasks)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 19, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"bg_block_\\\\d+\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"bg_block_\\\\d+\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"bg_block_\\\\d+\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"bg_block_\\\\d+\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Storage Background (Large Tasks)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Involved when manually compacting the data.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 20, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"m_compact_pool\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"m_compact_pool\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"m_compact_pool\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"m_compact_pool\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Manual Compaction", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 21, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"async_poller.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"async_poller.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"async_poller.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"async_poller.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GRPC Async Server", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, "height": null, "hideTimeOverride": false, "id": 22, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"GRPCComp.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"GRPCComp.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"GRPCComp.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"GRPCComp.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "GRPC Async Client", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, "height": null, "hideTimeOverride": false, "id": 23, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"fap_builder.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"fap_builder.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"fap_builder.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"fap_builder.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FAP builder", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 }, "height": null, "hideTimeOverride": false, "id": 24, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"snap_sender.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"snap_sender.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"snap_sender.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"snap_sender.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Snapshot Sender", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 }, "height": null, "hideTimeOverride": false, "id": 25, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"segment_sched.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"segment_sched.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"segment_sched.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"segment_sched.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Segment Scheduler", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 26, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"LocalIndexPool*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "pool-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"LocalIndexPool*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"LocalIndexPool*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"LocalIndexPool*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Local Index Pool", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 27, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"SegmentReader.*\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"SegmentReader.*\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"SegmentReader.*\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count((\n tiflash_proxy_thread_cpu_seconds_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",name=~\"SegmentReader.*\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Segment Reader", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Threads CPU", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 28, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 29, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_proxy_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, state) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{state}}", "metric": "", "query": "sum((\n tiflash_proxy_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, state) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_proxy_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-total", "metric": "", "query": "sum((\n tiflash_proxy_threads_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads state", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 30, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (name, io, $additional_groupby) > 1024", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{name}}-{{io}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_threads_io_bytes_total\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (name, io, $additional_groupby) > 1024", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads IO", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 31, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_voluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, name) > 200", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} - {{name}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_voluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, name) > 200", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Thread Voluntary Context Switches", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 32, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_thread_nonvoluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, name) > 50", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}} - {{name}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_thread_nonvoluntary_context_switches\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, name) > 50", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Thread Nonvoluntary Context Switches", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Threads", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 33, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 34, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_coprocessor_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_coprocessor_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 35, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_coprocessor_executor_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Executor QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 36, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_coprocessor_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_coprocessor_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_coprocessor_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_coprocessor_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 37, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (reason) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{reason}}", "metric": "", "query": "sum(rate(\n tiflash_coprocessor_request_error\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (reason) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Error QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 38, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_coprocessor_request_handle_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_coprocessor_request_handle_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_coprocessor_request_handle_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_coprocessor_request_handle_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request Handle Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 39, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_coprocessor_response_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Response Bytes/Seconds", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 40, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_coprocessor_request_memory_usage_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "999-{{type}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_coprocessor_request_memory_usage_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_coprocessor_request_memory_usage_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99-{{type}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_coprocessor_request_memory_usage_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.95,(\n sum(rate(\n tiflash_coprocessor_request_memory_usage_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "95-{{type}}", "metric": "", "query": "histogram_quantile(0.95,(\n sum(rate(\n tiflash_coprocessor_request_memory_usage_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_coprocessor_request_memory_usage_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "80-{{type}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_coprocessor_request_memory_usage_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Cop task memory usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 41, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_exchange_data_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_exchange_data_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Exchange Bytes/Seconds", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 42, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_thread_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"rpc.*\",type!~\".*max\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "max((\n tiflash_thread_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"rpc.*\",type!~\".*max\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads of Rpc", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 43, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_coprocessor_handling_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum((\n tiflash_coprocessor_handling_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Handling Request Number", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, "height": null, "hideTimeOverride": false, "id": 44, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_thread_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\".*max\",type!~\"rpc.*\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "max((\n tiflash_thread_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\".*max\",type!~\"rpc.*\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, "height": null, "hideTimeOverride": false, "id": 45, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_thread_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"rpc.*\",type=~\".*max\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "max((\n tiflash_thread_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"rpc.*\",type=~\".*max\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max Threads of Rpc", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The MPP query count in TiFlash", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 }, "height": null, "hideTimeOverride": false, "id": 46, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_mpp_task_manager\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "max((\n tiflash_mpp_task_manager\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "MPP Query count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 }, "height": null, "hideTimeOverride": false, "id": 47, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_thread_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\".*max\",type!~\"rpc.*\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "max((\n tiflash_thread_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\".*max\",type!~\"rpc.*\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Max Threads", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 48, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_mpp_task_monitor\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tiflash_mpp_task_monitor\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Time of the Longest Live MPP Task", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 49, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_exchange_queueing_data_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "((\n tiflash_exchange_queueing_data_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Data size in send and receive queue", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 64 }, "height": null, "hideTimeOverride": false, "id": 50, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_network_transmission_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_network_transmission_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Network Transmission", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The establish calldata details", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 64 }, "height": null, "hideTimeOverride": false, "id": 51, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_establish_calldata_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type != \"new_request_calldata\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "max((\n tiflash_establish_calldata_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type != \"new_request_calldata\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Establish calldata details", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Coprocessor", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 52, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "the min_tso of each instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 53, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 1, "points": true, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"min_tso\"}\n \n)) by (instance, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"min_tso\"}\n \n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Min TSO", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": "TSO", "logBase": 1, "max": null, "min": null, "show": false }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "estimated thread usage in min-tso scheduler, and the sort/hard limit of estimated thread in scheduler.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 54, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"thread_soft_limit\"}\n \n)) by (instance, type, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"thread_soft_limit\"}\n \n)) by (instance, type, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"estimated_thread_usage\"}\n \n)) by (instance, type, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"estimated_thread_usage\"}\n \n)) by (instance, type, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"thread_hard_limit\"}\n \n)) by (instance, type, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"thread_hard_limit\"}\n \n)) by (instance, type, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"global_estimated_thread_usage\"}\n \n)) by (instance, type, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"global_estimated_thread_usage\"}\n \n)) by (instance, type, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"group_entry_count\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"group_entry_count\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Estimated Thread Usage and Limit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": "Threads", "logBase": 10, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "the count of active/ waiting queries", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 55, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"waiting_queries_count\"}\n \n)) by (instance, type, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"waiting_queries_count\"}\n \n)) by (instance, type, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"active_queries_count\"}\n \n)) by (instance, type, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"active_queries_count\"}\n \n)) by (instance, type, resource_group) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Active and Waiting Queries Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": "Queries", "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "the count of active/ waiting tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 56, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"waiting_tasks_count\"}\n \n)) by (instance, type, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"waiting_tasks_count\"}\n \n)) by (instance, type, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"active_tasks_count\"}\n \n)) by (instance, type, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"active_tasks_count\"}\n \n)) by (instance, type, resource_group) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Active and Waiting Tasks Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": "Tasks", "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "the usage of estimated threads exceeded the hard limit where errors occur.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 57, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"hard_limit_exceeded_count\"}\n \n)) by (instance, type, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_task_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"hard_limit_exceeded_count\"}\n \n)) by (instance, type, resource_group) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Hard Limit Exceeded Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "the time of waiting for schedule", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 58, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, instance, resource_group, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{resource_group}}-max", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_task_scheduler_waiting_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, instance, resource_group, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, instance, resource_group, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{resource_group}}-9999", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, instance, resource_group, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, instance, resource_group, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{resource_group}}-999", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, instance, resource_group, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, instance, resource_group, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{resource_group}}-99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, instance, resource_group, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, instance, resource_group, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{resource_group}}-80", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, instance, resource_group, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, resource_group, $additional_groupby) / sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, resource_group, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{resource_group}}-avg", "metric": "", "query": "(sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, resource_group, $additional_groupby) / sum(rate(\n tiflash_task_scheduler_waiting_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, resource_group, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Waiting Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Task Scheduler", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 59, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Executed DDL jobs per minute", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 60, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (type)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "avg(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (type)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total", "metric": "", "query": "sum(increase(tiflash_schema_internal_ddl_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tiflash_schema_internal_ddl_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(increase(\n tiflash_schema_internal_ddl_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tiflash_schema_internal_ddl_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "total-{{instance}}", "metric": "", "query": "sum(increase(\n tiflash_schema_internal_ddl_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Schema Internal DDL OPM", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Executed DDL apply jobs per minute", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 61, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg(increase(tiflash_schema_trigger_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (type)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "triggle-by-{{type}}", "metric": "", "query": "avg(increase(tiflash_schema_trigger_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (type)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Schema Apply OPM", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 62, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^applying/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_schema_apply_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_schema_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_schema_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_schema_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_schema_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_schema_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_schema_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_schema_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_schema_apply_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_schema_apply_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_schema_apply_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_schema_apply_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_schema_apply_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_sync_schema_applying\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"$type\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "applying-{{instance}}", "metric": "", "query": "sum((\n tiflash_sync_schema_applying\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"$type\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Schema Apply Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "DDL", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 63, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "TiFlash CPU usage calculated with process CPU running seconds.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 64, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/limit/", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "irate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=~\".*tiflash\", instance=~\"$tiflash_role\"}[$__rate_interval])", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "irate(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", job=~\".*tiflash\", instance=~\"$tiflash_role\"}[$__rate_interval])", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_LogicalCPUCores\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "limit-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_LogicalCPUCores\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "CPU Usage (irate)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 65, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"SegmentReader.*\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"SegmentReader.*\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"SegmentReader.*\", instance=~\"$tiflash_role\"}) by (instance)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"SegmentReader.*\", instance=~\"$tiflash_role\"}) by (instance)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Segment Reader", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 66, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_coprocessor_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_coprocessor_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request QPS by instance", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The flow of different kinds of read operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 67, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "File Descriptor-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_PSMReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Page-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_PSMReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_PSMBackgroundReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "PageBackGround-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_PSMBackgroundReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read Throughput by instance", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total count of different kinds of commands received", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 68, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/delete_range|ingest/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_DMWriteBlock\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write block-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_DMWriteBlock\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tiflash_storage_command_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(increase(\n tiflash_storage_command_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Command OPS By Instance", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The throughput of write by instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 69, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/total/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"write\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"write\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "ingest-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Throughput By Instance", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Imbalance read/write", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 70, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 71, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_NumKeyspace\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "keyspace-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_NumKeyspace\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Number of Keyspaces", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 72, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_NumStorageDeltaMerge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "tables-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_NumStorageDeltaMerge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_NumIStorage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "tables-all-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_NumIStorage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Number of Physical Tables", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 73, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_NumSegment\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "segments-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_NumSegment\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_NumMemTable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "mem_table-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_NumMemTable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Number of Segments", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 74, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_BytesMemTable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "bytes-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_BytesMemTable\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_BytesMemTableAllocated\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "bytes-allocated-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_BytesMemTableAllocated\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Bytes of MemTables", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The memory usage of mark cache and minmax index cache", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 75, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/limit/", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_asynchronous_metric_MarkCacheBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "mark_cache_{{instance}}", "metric": "", "query": "((\n tiflash_system_asynchronous_metric_MarkCacheBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_asynchronous_metric_MinMaxIndexFiles\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "minmax_index_cache_{{instance}}", "metric": "", "query": "((\n tiflash_system_asynchronous_metric_MinMaxIndexFiles\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_asynchronous_metric_RNMVCCIndexCacheBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "rn_mvcc_index_cache_{{instance}}", "metric": "", "query": "((\n tiflash_system_asynchronous_metric_RNMVCCIndexCacheBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Mark Cache and Minmax Index Cache Memory Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "cache misses or cache hits of mark_cache.\nBased on this infactor, we can check whether mark_cache is large enough", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 76, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_system_profile_event_MarkCacheMisses\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "mark cache misses", "metric": "", "query": "max((\n tiflash_system_profile_event_MarkCacheMisses\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_system_profile_event_MarkCacheHits\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "mark cache hits", "metric": "", "query": "max((\n tiflash_system_profile_event_MarkCacheHits\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Effectiveness of Mark Cache", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Information about schema of column file, to learn the memory usage of schema", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 77, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_shared_block_schemas\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"current_size\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "current_size-{{instance}}", "metric": "", "query": "max((\n tiflash_shared_block_schemas\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"current_size\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_shared_block_schemas\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"hit_count\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "hit_count_ops-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_shared_block_schemas\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"hit_count\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_shared_block_schemas\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"still_used_when_evict\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "still_used_when_evict-{{instance}}", "metric": "", "query": "max((\n tiflash_shared_block_schemas\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"still_used_when_evict\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_shared_block_schemas\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"miss_count\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "miss_count_ops-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_shared_block_schemas\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"miss_count\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Schema of Column File", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 78, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/max_snapshot_lifetime/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_SegmentReadTasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read_tasks-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_SegmentReadTasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max_snapshot_lifetime-{{instance}}", "metric": "", "query": "((\n tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read Snapshots", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 79, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "rate(tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"alloc_.*\"}[$__interval])", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "rate(tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"alloc_.*\"}[$__interval])", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "-rate(tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}[$__interval])", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "-rate(tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}[$__interval])", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_storages_thread_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"alloc_.*\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-tot", "metric": "", "query": "((\n tiflash_storages_thread_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"alloc_.*\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "-tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-tot", "metric": "", "query": "-tiflash_storages_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory by thread", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 80, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "rate(tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"alloc_.*\"}[$__interval])", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "rate(tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"alloc_.*\"}[$__interval])", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "-rate(tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}[$__interval])", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "-rate(tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}[$__interval])", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_raft_proxy_thread_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"alloc_.*\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-tot", "metric": "", "query": "((\n tiflash_raft_proxy_thread_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"alloc_.*\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "-tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-tot", "metric": "", "query": "-tiflash_raft_proxy_thread_memory_usage{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dealloc_.*\"}", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory by thread (proxy)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, "height": null, "hideTimeOverride": false, "id": 81, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_memory_usage_by_class\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "((\n tiflash_memory_usage_by_class\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "rate(tiflash_memory_usage_by_class{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__interval])", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}-rate", "metric": "", "query": "rate(tiflash_memory_usage_by_class{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__interval])", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory by class", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, "height": null, "hideTimeOverride": false, "id": 82, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_MemoryTrackingKVStore\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_MemoryTrackingKVStore\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "KVStore memory", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Memory trace", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 83, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 84, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/limit/", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_proxy_kv_engine_ia_main_queue_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "capacity-main-{{instance}}", "metric": "", "query": "((\n tiflash_proxy_kv_engine_ia_main_queue_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_proxy_kv_engine_ia_small_queue_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "capacity-small-{{instance}}", "metric": "", "query": "((\n tiflash_proxy_kv_engine_ia_small_queue_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_proxy_kv_engine_ia_manager_segments_memory_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "capacity-segments-{{instance}}", "metric": "", "query": "((\n tiflash_proxy_kv_engine_ia_manager_segments_memory_capacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_proxy_kv_engine_ia_manager_segments_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "segments-mem-size-{{instance}}", "metric": "", "query": "((\n tiflash_proxy_kv_engine_ia_manager_segments_memory_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_proxy_kv_engine_ia_manager_segments_disk_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "segments-disk-size-{{instance}}", "metric": "", "query": "((\n tiflash_proxy_kv_engine_ia_manager_segments_disk_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 85, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_kv_engine_ia_manager_segments_memory_wait_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA Segments Memory Wait", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 86, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "cache-hit {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "cache-miss {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA Segment Remote Read Cache", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 87, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_kv_engine_ia_remote_read_segment_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "IA Segments Remote Read Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 88, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_kv_engine_columnar_file_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "file-cache-hit {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_kv_engine_columnar_file_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_kv_engine_columnar_file_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "file-cache-miss {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_kv_engine_columnar_file_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "ColumnarFile Cache", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 89, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Columnar Prefetch Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 90, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_kv_engine_columnar_prefetch_cache_hit_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Columnar Prefetch Cache Hit Ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percent", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 91, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_retry_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "retry {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_retry_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Columnar Fetch Snapshot Retry", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 92, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_proxy_kv_engine_columnar_fetch_snapshot_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Columnar Fetch Snapshot Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 93, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_kv_engine_columnar_meta_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "hit {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_kv_engine_columnar_meta_cache_hit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_kv_engine_columnar_meta_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "miss {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_kv_engine_columnar_meta_cache_miss\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_proxy_kv_engine_columnar_meta_cache_parse\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "parse {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_proxy_kv_engine_columnar_meta_cache_parse\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Columnar Meta Cache", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 94, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_proxy_kv_engine_columnar_meta_cache_entries\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "entries-{{instance}}", "metric": "", "query": "((\n tiflash_proxy_kv_engine_columnar_meta_cache_entries\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Columnar Meta Cache Entries", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 95, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_proxy_kv_engine_columnar_meta_cache_weighted_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "weighted_size-{{instance}}", "metric": "", "query": "((\n tiflash_proxy_kv_engine_columnar_meta_cache_weighted_size\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Columnar Meta Cache Weighted Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Columnar Storage", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 96, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total count of different kinds of commands received", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 97, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/delete_range|ingest/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tiflash_storage_command_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(increase(\n tiflash_storage_command_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_DMWriteBlock\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write block", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_DMWriteBlock\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Command OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 98, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/fs|write/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum by (instance) ( tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"} + tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"} ) / sum by (instance) ( tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"} )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "amp-total-{{instance}}", "metric": "", "query": "sum by (instance) ( tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"} + tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"} ) / sum by (instance) ( tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"} )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum by (instance) ( rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]) ) / sum by (instance) ( rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[5m]) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "amp-5min-{{instance}}", "metric": "", "query": "sum by (instance) ( rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]) ) / sum by (instance) ( rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[5m]) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum by (instance) ( rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[10m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[10m]) ) / sum by (instance) ( rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[10m]) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "amp-10min-{{instance}}", "metric": "", "query": "sum by (instance) ( rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[10m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[10m]) ) / sum by (instance) ( rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[10m]) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum by (instance) ( rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[30m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[30m]) ) / sum by (instance) ( rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[30m]) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "amp-30min-{{instance}}", "metric": "", "query": "sum by (instance) ( rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[30m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[30m]) ) / sum by (instance) ( rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[30m]) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum by (instance) ( rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "fs-5min-{{instance}}", "metric": "", "query": "sum by (instance) ( rate(tiflash_system_profile_event_PSMWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]) + rate(tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[5m]) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum by (instance) ( rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[5m]) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write-5min-{{instance}}", "metric": "", "query": "sum by (instance) ( rate(tiflash_storage_throughput_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"write|ingest\"}[5m]) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Amplification", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": "20", "min": "0", "show": true }, { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The throughput of (maybe foreground) tasks of storage in bytes", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 99, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/total/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_subtask_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_storage_subtask_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "SubTasks Write Throughput (bytes)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The throughput of (maybe foreground) tasks of storage in rows", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 100, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/total/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_subtask_throughput_rows\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_storage_subtask_throughput_rows\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "SubTasks Write Throughput (rows)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total number of storage's internal sub tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 101, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_subtask_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_storage_subtask_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Small Internal Tasks OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of storage's internal sub tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 102, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_subtask_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_subtask_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_subtask_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_subtask_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Small Internal Tasks Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total number of storage's internal sub tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 103, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_subtask_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_storage_subtask_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Large Internal Tasks OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of storage's internal sub tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 21 }, "height": null, "hideTimeOverride": false, "id": 104, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_subtask_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_subtask_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_subtask_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_subtask_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"(delta_merge|seg_merge|seg_split).*\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Large Internal Tasks Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The current processing number of segments' background management", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 26 }, "height": null, "hideTimeOverride": false, "id": 105, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tiflash_system_current_metric_DT_DeltaMerge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "delta_merge-{{instance}}", "metric": "", "query": "avg((\n tiflash_system_current_metric_DT_DeltaMerge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tiflash_system_current_metric_DT_SegmentSplit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seg_split-{{instance}}", "metric": "", "query": "avg((\n tiflash_system_current_metric_DT_SegmentSplit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tiflash_system_current_metric_DT_SegmentMerge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seg_merge-{{instance}}", "metric": "", "query": "avg((\n tiflash_system_current_metric_DT_SegmentMerge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Current Data Management Tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of currently opened file descriptors.\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 34 }, "height": null, "hideTimeOverride": false, "id": 106, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_proxy_process_open_fds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "((\n tiflash_proxy_process_open_fds\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\",job=~\".*tiflash\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_OpenFileForWrite\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "W-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_OpenFileForWrite\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_OpenFileForRead\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "R-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_OpenFileForRead\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_OpenFileForReadWrite\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "RW-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_OpenFileForReadWrite\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Opened File Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of open file descriptors action.\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 34 }, "height": null, "hideTimeOverride": false, "id": 107, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_FileOpen\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Open-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_FileOpen\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_FileOpenFailed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "OpenFail-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_FileOpenFailed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "File Open OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "OPS and duration of fsync operations.\n(Only counting storage engine of TiFlash by now. Not including TiFlash-Proxy)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 34 }, "height": null, "hideTimeOverride": false, "id": 108, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/max-fsync/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_FileFSync\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "ops-fsync-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_FileFSync\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_system_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"fsync\"}[$__rate_interval]))) by (le, instance) / 1000000000)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-fsync-{{instance}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_system_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"fsync\"}[$__rate_interval]))) by (le, instance) / 1000000000)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FSync Status", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of different kinds of read operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 109, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_PSMWriteIOCalls\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Page", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_PSMWriteIOCalls\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_PSMWritePages\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "PageFile", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_PSMWritePages\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "File Descriptor", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_WriteBufferFromFileDescriptorWrite\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk Write OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of different kinds of read operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 42 }, "height": null, "hideTimeOverride": false, "id": 110, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_PSMReadIOCalls\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Page", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_PSMReadIOCalls\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_PSMReadPages\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "PageFile", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_PSMReadPages\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_ReadBufferFromFileDescriptorRead\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "File Descriptor", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_ReadBufferFromFileDescriptorRead\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Disk Read OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The flow of different kinds of write operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 50 }, "height": null, "hideTimeOverride": false, "id": 111, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "File Descriptor", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_WriteBufferFromFileDescriptorWriteBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_PSMWriteBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Page", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_PSMWriteBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_PSMBackgroundWriteBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "PageBackGround", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_PSMBackgroundWriteBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The flow of different kinds of read operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 50 }, "height": null, "hideTimeOverride": false, "id": 112, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "File Descriptor", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_ReadBufferFromFileDescriptorReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_PSMReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Page", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_PSMReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_PSMBackgroundReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "PageBackGround", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_PSMBackgroundReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The compression ratio of different compression algorithm", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 58 }, "height": null, "hideTimeOverride": false, "id": 113, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": true, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lz4_uncompressed_bytes\"}[$__rate_interval]))/sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lz4_compressed_bytes\"}[$__rate_interval]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "lz4", "metric": "", "query": "sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lz4_uncompressed_bytes\"}[$__rate_interval]))/sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lz4_compressed_bytes\"}[$__rate_interval]))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lightweight_uncompressed_bytes\"}[$__rate_interval]))/sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lightweight_compressed_bytes\"}[$__rate_interval]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "lightweight", "metric": "", "query": "sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lightweight_uncompressed_bytes\"}[$__rate_interval]))/sum(rate(tiflash_storage_pack_compression_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"lightweight_compressed_bytes\"}[$__rate_interval]))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compression Ratio", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The count of the compression algorithm used by each data part", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 58 }, "height": null, "hideTimeOverride": false, "id": 114, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": true, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_pack_compression_algorithm_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_storage_pack_compression_algorithm_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Compression Algorithm Count", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Storage", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 115, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total number of storage engine read tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 116, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_read_tasks_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_storage_read_tasks_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read Tasks OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 117, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/max_snapshot_lifetime/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_SegmentReadTasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read_tasks-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_SegmentReadTasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_PSMVCCSnapshotsList\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "snapshot_list-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_PSMVCCSnapshotsList\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_PSMVCCNumSnapshots\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "num_snapshot-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_PSMVCCNumSnapshots\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_SnapshotOfRead\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_SnapshotOfRead\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_SnapshotOfReadRaw\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "read_raw-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_SnapshotOfReadRaw\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_SnapshotOfDeltaMerge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "delta_merge-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_SnapshotOfDeltaMerge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_SnapshotOfDeltaCompact\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "delta_compact-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_SnapshotOfDeltaCompact\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_SnapshotOfSegmentMerge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seg_merge-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_SnapshotOfSegmentMerge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_SnapshotOfSegmentSplit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "seg_split-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_SnapshotOfSegmentSplit\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_current_metric_DT_SnapshotOfPlaceIndex\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "place_index-{{instance}}", "metric": "", "query": "((\n tiflash_system_current_metric_DT_SnapshotOfPlaceIndex\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max_snapshot_lifetime-{{instance}}", "metric": "", "query": "((\n tiflash_system_asynchronous_metric_MaxDTDeltaOldestSnapshotLifetime\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read Snapshots", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 118, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_read_thread_internal_us_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_read_thread_internal_us_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_read_thread_internal_us_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_read_thread_internal_us_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_read_thread_internal_us_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_read_thread_internal_us_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_read_thread_internal_us_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_read_thread_internal_us_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_read_thread_internal_us_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_read_thread_internal_us_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_read_thread_internal_us_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_read_thread_internal_us_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_read_thread_internal_us_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read Thread Internal Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "\u00b5s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The information of read thread scheduling.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 119, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/push_block/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_read_thread_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"ru_exhausted|sche_active_segment_limit|sche_from_cache|sche_new_task|sche_no_pool|sche_no_ru|sche_no_segment|sche_no_slot|push_block_bytes\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_storage_read_thread_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"ru_exhausted|sche_active_segment_limit|sche_from_cache|sche_new_task|sche_no_pool|sche_no_ru|sche_no_segment|sche_no_slot|push_block_bytes\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read Thread Scheduling", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The information of data sharing cache hit ratio. Data sharing cache is purpose-built for OLAP workload that can reduce repeated data reads of concurrent table scanning.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 120, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/cache_hit_ratio/", "yaxis": 2 }, { "alias": "/cache_hit_ratio/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_read_thread_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"add_cache_total_bytes_limit\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_storage_read_thread_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"add_cache_total_bytes_limit\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"data_sharing_hit\"}[$__rate_interval])) / sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"data_sharing_hit|data_sharing_miss\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "data_sharing_cache_hit_ratio", "metric": "", "query": "(sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"data_sharing_hit\"}[$__rate_interval])) / sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"data_sharing_hit|data_sharing_miss\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"extra_column_hit\"}[$__rate_interval])) / sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"extra_column_hit|extra_column_miss\"}[$__rate_interval])))", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "extra_column_cache_hit_ratio", "metric": "", "query": "(sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"extra_column_hit\"}[$__rate_interval])) / sum(rate(tiflash_storage_column_cache_packs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"extra_column_hit|extra_column_miss\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Data Sharing", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 121, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/cache_hit_ratio/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_read_thread_gauge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum((\n tiflash_storage_read_thread_gauge\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Segment MergedTask", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 122, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_read_thread_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_read_thread_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_read_thread_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_read_thread_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_read_thread_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_read_thread_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_read_thread_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_read_thread_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_read_thread_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_read_thread_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_read_thread_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_read_thread_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_read_thread_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_read_thread_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Segment MergedTask Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 123, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_version_chain_ms_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_version_chain_ms_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_version_chain_ms_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_version_chain_ms_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_version_chain_ms_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_version_chain_ms_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_version_chain_ms_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_version_chain_ms_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_version_chain_ms_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_version_chain_ms_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_version_chain_ms_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_version_chain_ms_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_version_chain_ms_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_version_chain_ms_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "VersionChain", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ms", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Errors of DeltaIndex", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 124, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_DTDeltaIndexError\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "DeltaIndexError-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_DTDeltaIndexError\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "DeltaIndexError", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Storage Read Pool & Data Sharing", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 125, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The disk usage of PageStorage instances in each TiFlash node", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 126, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^valid_rate/", "yaxis": 2 }, { "alias": "/size/", "linewidth": 3 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_asynchronous_metric_BlobDiskBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "blob_disk_size-{{instance}}", "metric": "", "query": "((\n tiflash_system_asynchronous_metric_BlobDiskBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_BlobValidBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "blob_valid_size-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_BlobValidBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) / (tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})) by (instance)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "blob_valid_rate-{{instance}}", "metric": "", "query": "sum((tiflash_system_asynchronous_metric_BlobValidBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}) / (tiflash_system_asynchronous_metric_BlobDiskBytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"})) by (instance)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_asynchronous_metric_LogDiskBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "log_size-{{instance}}", "metric": "", "query": "((\n tiflash_system_asynchronous_metric_LogDiskBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PageStorage Disk Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": "1.1", "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The number of files of PageStorage instances in each TiFlash node", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 127, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_BlobFileNums\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "blob_file-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_BlobFileNums\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_asynchronous_metric_LogNums\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "log_file-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_asynchronous_metric_LogNums\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PageStorage File Num", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 128, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_storage_page_write_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"v3\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_storage_page_write_batch_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"v3\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "PageStorage WriteBatch Size", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 129, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_write_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_page_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_page_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_page_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_page_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_page_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_page_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_page_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_page_write_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_page_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_page_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_page_write_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_page_write_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Page write Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 130, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tiflash_storage_page_gc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "sum(increase(\n tiflash_storage_page_gc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Page GC Tasks OPM", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 131, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_page_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_page_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_page_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_page_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_page_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_page_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_page_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_page_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_page_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_page_gc_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_page_gc_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_page_gc_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_page_gc_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Page GC Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The number of pages of all TiFlash instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 132, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_asynchronous_metric_PagesInMem\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "num_pages-{{instance}}", "metric": "", "query": "((\n tiflash_system_asynchronous_metric_PagesInMem\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_system_asynchronous_metric_VersionedEntries\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "num_entries-{{instance}}", "metric": "", "query": "((\n tiflash_system_asynchronous_metric_VersionedEntries\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Numer of Pages", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The num of pending writers in PageStorage", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 133, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_PSPendingWriterNum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "size-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_PSPendingWriterNum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PageStorage Pending Writers Num", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 134, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_page_data_by_types\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum((\n tiflash_storage_page_data_by_types\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PageStorage stored bytes by type", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The number of tables running under different mode in DeltaTree", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 135, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_StoragePoolV2Only\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "V2-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_StoragePoolV2Only\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_StoragePoolV3Only\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "V3-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_StoragePoolV3Only\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_StoragePoolMixMode\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Mix-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_StoragePoolMixMode\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_StoragePoolUniPS\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "UniPS-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_StoragePoolUniPS\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Number of Tables", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 40 }, "height": null, "hideTimeOverride": false, "id": 136, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_page_command_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_storage_page_command_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PS Command OPS By Instance", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 48 }, "height": null, "hideTimeOverride": false, "id": 137, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_page_apply_edit_type\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_storage_page_apply_edit_type\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PS Apply edits OPS By Instance", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "PageStorage", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 138, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage I/O limiter metrics.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 139, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_io_limiter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_storage_io_limiter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "I/O Limiter Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Current limit bytes per second of Storage I/O limiter", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 140, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_io_limiter_curr\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum((\n tiflash_storage_io_limiter_curr\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "I/O Limiter Threshold", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "I/O Limiter current pending gauge.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 141, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/pending/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tiflash_system_current_metric_RateLimiterPendingWriteRequest\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "other-current-{{instance}}", "metric": "", "query": "avg((\n tiflash_system_current_metric_RateLimiterPendingWriteRequest\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tiflash_system_current_metric_IOLimiterPendingBgWriteReq\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "bgwrite-current-{{instance}}", "metric": "", "query": "avg((\n tiflash_system_current_metric_IOLimiterPendingBgWriteReq\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tiflash_system_current_metric_IOLimiterPendingFgWriteReq\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "fgwrite-current-{{instance}}", "metric": "", "query": "avg((\n tiflash_system_current_metric_IOLimiterPendingFgWriteReq\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tiflash_system_current_metric_IOLimiterPendingBgReadReq\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "bgread-current-{{instance}}", "metric": "", "query": "avg((\n tiflash_system_current_metric_IOLimiterPendingBgReadReq\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((\n tiflash_system_current_metric_IOLimiterPendingFgReadReq\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "fgread-current-{{instance}}", "metric": "", "query": "avg((\n tiflash_system_current_metric_IOLimiterPendingFgReadReq\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "I/O Limiter Current Pending Gauge", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The storage I/O limiter metrics.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 142, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_io_limiter_pending_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_storage_io_limiter_pending_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "I/O Limiter Pending OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "I/O Limiter pending duration.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 143, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-pending-max", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_io_limiter_pending_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_io_limiter_pending_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-pending-9999", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_io_limiter_pending_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_io_limiter_pending_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-pending-999", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_io_limiter_pending_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_io_limiter_pending_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-pending-99", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_io_limiter_pending_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_io_limiter_pending_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-pending-80", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_io_limiter_pending_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_io_limiter_pending_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_io_limiter_pending_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-pending-avg", "metric": "", "query": "(sum(rate(\n tiflash_storage_io_limiter_pending_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_io_limiter_pending_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "I/O Limiter Pending Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Rate Limiter", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 144, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The stall duration of write and delete range", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 145, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "99-delta_merge", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, instance, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}}-{{instance}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_write_stall_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, instance, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_write_stall_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, instance, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_write_stall_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, instance, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_write_stall_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, instance, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_write_stall_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, instance, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_write_stall_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, instance, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_write_stall_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, instance, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_write_stall_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, instance, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}}-{{instance}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_write_stall_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, instance, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_write_stall_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance, $additional_groupby) / sum(rate(\n tiflash_storage_write_stall_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}}-{{instance}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_write_stall_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance, $additional_groupby) / sum(rate(\n tiflash_storage_write_stall_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Stall Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The throughput of write and delta's background management", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 146, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"write|ingest\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write+ingest", "metric": "", "query": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"write|ingest\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"write|ingest\"}\n [$__rate_interval]\n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "ManageDelta", "metric": "", "query": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"write|ingest\"}\n [$__rate_interval]\n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write & Delta Management Throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The throughput of write and delta's background management", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 147, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"write|ingest\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write+ingest", "metric": "", "query": "sum((\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"write|ingest\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"write|ingest\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "ManageDelta", "metric": "", "query": "sum((\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type!~\"write|ingest\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write & Delta Management Total", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The throughput of write by instance", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 148, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 250, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/total/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"write\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"write\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "ingest-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_storage_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"ingest\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Throughput By Instance", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The total count of different kinds of commands received", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 149, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/delete_range|ingest/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_DMWriteBlock\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "write block-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_DMWriteBlock\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tiflash_storage_command_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(increase(\n tiflash_storage_command_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Write Command OPS By Instance", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Storage Write Stall", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 150, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 151, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_stale_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_stale_read_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Stale Read OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 152, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_raft_read_index_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_raft_read_index_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Read Index OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 153, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_raft_learner_read_failures_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_raft_learner_read_failures_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Learner Read Failures", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 154, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_raft_read_index_events_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_raft_read_index_events_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read Index Events", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 155, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/timeout/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_wait_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_wait_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_wait_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_wait_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_wait_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_wait_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_wait_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_wait_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_wait_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_raft_wait_index_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_raft_wait_index_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_raft_wait_index_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_raft_wait_index_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(\n tiflash_system_profile_event_RaftWaitIndexTimeout\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-timeout", "metric": "", "query": "sum(increase(\n tiflash_system_profile_event_RaftWaitIndexTimeout\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Wait Index Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "opm", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of currently applying snapshots.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 156, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_read_index_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_read_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_read_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_read_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_read_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_read_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_read_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_read_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_read_index_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_raft_read_index_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_raft_read_index_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_raft_read_index_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_raft_read_index_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Batch Read Index Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of applying Raft write logs", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 157, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_apply_write_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_raft_apply_write_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"write\"}[$__rate_interval])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"write\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-write", "metric": "", "query": "(sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"write\"}[$__rate_interval])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"write\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"admin\"}[$__rate_interval])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"admin\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-admin", "metric": "", "query": "(sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"admin\"}[$__rate_interval])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"admin\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"flush_region\"}[$__rate_interval])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"flush_region\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-flush_region", "metric": "", "query": "(sum(rate(tiflash_raft_apply_write_command_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"flush_region\"}[$__rate_interval])) / sum(rate(tiflash_raft_apply_write_command_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"flush_region\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"decode\"}[$__rate_interval])) / sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"decode\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-decode", "metric": "", "query": "(sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"decode\"}[$__rate_interval])) / sum(rate(tiflash_raft_write_data_to_storage_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"decode\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Apply Raft write logs Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of decoding Region data into blocks when writing Region data to the storage layer. (Mixed with \"write logs\" and \"apply Snapshot\" operations)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 158, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_write_data_to_storage_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"decode\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_write_data_to_storage_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"decode\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region write Duration (decode)", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of writing Region data blocks to the storage layer (Mixed with \"write logs\" and \"apply Snapshot\" operations)", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 159, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_write_data_to_storage_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_write_data_to_storage_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region write Duration (write blocks)", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of applying Raft write logs", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 160, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"write\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"write\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply Raft write logs Duration [Heatmap]", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of applying Raft write logs", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 161, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"admin\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_apply_write_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"admin\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Apply Raft admin logs Duration [Heatmap]", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 }, "height": null, "hideTimeOverride": false, "id": 162, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_raft_raft_events_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_raft_raft_events_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Events QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 }, "height": null, "hideTimeOverride": false, "id": 163, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_raft_raft_frequent_events_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_raft_raft_frequent_events_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Frequent Events QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 164, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_raft_log_gap_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"applied_index\"}\n [$__rate_interval]\n)) by (le, type) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_raft_log_gap_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"applied_index\"}\n [$__rate_interval]\n)) by (le, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_raft_log_gap_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"compact_index\"}\n [$__rate_interval]\n)) by (le, type) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_raft_log_gap_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"compact_index\"}\n [$__rate_interval]\n)) by (le, type) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Raft Log Gap Heatmap", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 165, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_entry_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"normal\"}\n [$__rate_interval]\n)) by (le, type) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_entry_size_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"normal\"}\n [$__rate_interval]\n)) by (le, type) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Raft Entry Batch Size Heatmap", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 64 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 166, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_region_flush_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"unflushed\"}\n [$__rate_interval]\n)) by (le, type) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_region_flush_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"unflushed\"}\n [$__rate_interval]\n)) by (le, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_region_flush_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"flushed\"}\n [$__rate_interval]\n)) by (le, type) ", "format": "heatmap", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_region_flush_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"flushed\"}\n [$__rate_interval]\n)) by (le, type) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Region Size (by event) Heatmap", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 64 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 167, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_write_flow_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"big_write_to_region\"}\n [$__rate_interval]\n)) by (le, type) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_write_flow_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"big_write_to_region\"}\n [$__rate_interval]\n)) by (le, type) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Big Write To Region Size Heatmap", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 72 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 168, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_write_flow_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"write_committed\"}\n [$__rate_interval]\n)) by (le, type) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_write_flow_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"write_committed\"}\n [$__rate_interval]\n)) by (le, type) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Write Committed Size Heatmap", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 80 }, "height": null, "hideTimeOverride": false, "id": 169, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_raft_eager_gc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_raft_eager_gc_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Eager GC OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of Raft logs eager GC tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 80 }, "height": null, "hideTimeOverride": false, "id": 170, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_eager_gc_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_eager_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_eager_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_eager_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_eager_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_eager_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_eager_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_eager_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_eager_gc_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_raft_eager_gc_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_raft_eager_gc_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_raft_eager_gc_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_raft_eager_gc_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft Eager GC Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The keys flow of different kinds of Raft operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 88 }, "height": null, "hideTimeOverride": false, "id": 171, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_raft_process_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_raft_process_keys\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Keys flow", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 96 }, "height": null, "hideTimeOverride": false, "id": 172, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_raft_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_raft_throughput_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Raft throughput", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Latency that TiKV sends raft log to TiFlash.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 104 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 173, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_upstream_latency_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_upstream_latency_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Upstream Latency [Heatmap]", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Latency that TiKV sends raft log to TiFlash.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 104 }, "height": null, "hideTimeOverride": false, "id": 174, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_upstream_latency_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_upstream_latency_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_upstream_latency_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_upstream_latency_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_upstream_latency_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_upstream_latency_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_upstream_latency_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_upstream_latency_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_upstream_latency_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_raft_upstream_latency_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_raft_upstream_latency_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_raft_upstream_latency_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) / sum(rate(\n tiflash_raft_upstream_latency_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by ($additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Upstream Latency", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 112 }, "height": null, "hideTimeOverride": false, "id": 175, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_proxy_tikv_server_raft_append_rejects{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}", "metric": "", "query": "sum(rate(tiflash_proxy_tikv_server_raft_append_rejects{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Log Replication Rejected", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 176, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 177, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_raft_command_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_raft_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_raft_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_raft_command_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_raft_command_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Heavy Raft Apply Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of currently applying snapshots.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 178, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_RaftNumSnapshotsPendingApply\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Pending-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_RaftNumSnapshotsPendingApply\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_RaftNumPrehandlingSubTasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "PrehandleSubtasks-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_RaftNumPrehandlingSubTasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_RaftNumParallelPrehandlingTasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "ParallelTasks-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_RaftNumParallelPrehandlingTasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_RaftNumWaitedParallelPrehandlingTasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "Pending-ParallelTasks-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_RaftNumWaitedParallelPrehandlingTasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Applying snapshots Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 179, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_write_flow_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"snapshot_uncommitted\"}\n [$__rate_interval]\n)) by (le, type) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_write_flow_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"snapshot_uncommitted\"}\n [$__rate_interval]\n)) by (le, type) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Snapshot Uncommitted Size Heatmap", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 180, "interval": null, "isNew": true, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_raft_ongoing_snapshot_total_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(rate(\n tiflash_raft_ongoing_snapshot_total_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Ongoing raft snapshot", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 181, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_snapshot_total_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"approx_raft_snapshot\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_snapshot_total_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"approx_raft_snapshot\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Snapshot Size Heatmap", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of pre-decode when applying region snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 182, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"snapshot_predecode\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"snapshot_predecode\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Snapshot Predecode Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 183, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_command_throughput_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"prehandle_snapshot\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_command_throughput_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"prehandle_snapshot\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Snapshot Prehandle Throughput Heatmap", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of pre-decode when applying region snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 184, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"snapshot_flush\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"snapshot_flush\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Snapshot Flush Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 185, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_write_flow_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"ingest_uncommitted\"}\n [$__rate_interval]\n)) by (le, type) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_write_flow_bytes_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"ingest_uncommitted\"}\n [$__rate_interval]\n)) by (le, type) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Ingest Uncommitted Size Heatmap", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of SST to DT in pre-decode when applying region snapshot", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 186, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"snapshot_predecode_sst2dt\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"snapshot_predecode_sst2dt\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Snapshot Predecode SST to DT Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of ingesting SST", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 187, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"ingest_sst\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_raft_command_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"ingest_sst\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Ingest SST Duration", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Raft Snapshot / IngestSST", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 188, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 189, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^RS Filter/", "yaxis": 2 }, { "alias": "/^PK/", "yaxis": 2 }, { "alias": "/^No Filter/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (instance)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "1min-{{instance}}", "metric": "", "query": "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (instance)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (instance)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "5min-{{instance}}", "metric": "", "query": "avg((rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]) - rate(tiflash_system_profile_event_DMFileFilterAftRoughSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) / (rate(tiflash_system_profile_event_DMFileFilterAftPKAndPackSet{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (instance)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_DMFileFilterNoFilter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "No Filter-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_DMFileFilterNoFilter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_DMFileFilterAftPKAndPackSet\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "PK Filter-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_DMFileFilterAftPKAndPackSet\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_DMFileFilterAftRoughSet\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "RS Filter-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_DMFileFilterAftRoughSet\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Rough Set Filter Rate", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "cacheTimeout": null, "cards": { "cardPadding": null, "cardRound": null }, "color": { "cardColor": "#b4ff00", "colorScale": "sqrt", "colorScheme": "interpolateSpectral", "exponent": 0.5, "max": null, "min": null, "mode": "spectrum" }, "dataFormat": "tsbuckets", "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "heatmap": {}, "height": null, "hideTimeOverride": false, "hideZeroBuckets": true, "highlightCards": true, "id": 190, "interval": null, "legend": { "show": false }, "links": [], "maxDataPoints": 512, "maxPerRow": null, "minSpan": null, "options": {}, "repeat": null, "repeatDirection": null, "reverseYBuckets": false, "span": null, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(delta(\n tiflash_storage_rough_set_filter_rate_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le) ", "format": "heatmap", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{le}}", "metric": "", "query": "sum(delta(\n tiflash_storage_rough_set_filter_rate_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le) ", "refId": "", "step": 10, "target": "" } ], "timeFrom": null, "timeShift": null, "title": "Rough Set Filter Rate Histogram", "tooltip": { "msResolution": true, "shared": true, "show": true, "showHistogram": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "heatmap", "xAxis": { "mode": "time", "name": null, "show": true, "values": [] }, "xBucketNumber": null, "xBucketSize": null, "yAxis": { "decimals": null, "format": "percent", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, "yBucketBound": "upper", "yBucketNumber": null, "yBucketSize": null } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Rough Set Filter Rate Histogram", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 191, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "PageStorage Checkpoint Duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 192, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_checkpoint_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_checkpoint_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_checkpoint_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_checkpoint_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_checkpoint_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_checkpoint_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_checkpoint_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_checkpoint_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_checkpoint_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_checkpoint_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_checkpoint_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_checkpoint_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_checkpoint_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Checkpoint Upload Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The flow of checkpoint operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 193, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_checkpoint_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"incremental\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "incremental {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_checkpoint_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"incremental\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_checkpoint_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"compaction\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "compaction {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_checkpoint_flow\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"compaction\"}\n [$__rate_interval]\n)) by ($additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Checkpoint Upload flow", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "The keys of checkpoint operations. All keys are uploaded in the checkpoint. Grouped by key types.", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 194, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_checkpoint_keys_by_types\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_checkpoint_keys_by_types\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Checkpoint Upload keys speed by type (all)", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The flow of checkpoint operations. Group by key types", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 195, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_checkpoint_flow_by_types\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_checkpoint_flow_by_types\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Checkpoint Upload flow by type (incremental+compaction)", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The number of files of owned by each TiFlash node", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 196, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_remote_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"num_files\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "checkpoint_data-{{instance}}", "metric": "", "query": "sum((\n tiflash_storage_remote_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"num_files\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote File Num", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The remote store usage owned by each TiFlash node", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 197, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/^valid_rate/", "yaxis": 2 }, { "alias": "/size/", "linewidth": 3 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_remote_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"total_size\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "remote_size-{{instance}}", "metric": "", "query": "sum((\n tiflash_storage_remote_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"total_size\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_remote_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"valid_size\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "valid_size-{{instance}}", "metric": "", "query": "sum((\n tiflash_storage_remote_stats\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"valid_size\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"valid_size\"}) / (tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"total_size\"})) by (instance)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "valid_rate-{{instance}}", "metric": "", "query": "sum((tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"valid_size\"}) / (tiflash_storage_remote_stats{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"total_size\"})) by (instance)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Store Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": "1.1", "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 198, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_disaggregated_object_lock_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_disaggregated_object_lock_request_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Object Lock Request QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 199, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_disaggregated_object_lock_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_disaggregated_object_lock_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_disaggregated_object_lock_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Object Lock Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 200, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_s3_store_summary_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, store_id, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "store-{{store_id}}-{{type}}", "metric": "", "query": "sum((\n tiflash_storage_s3_store_summary_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, store_id, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Store Summary", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 8, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 201, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/total/", "yaxis": 2 }, { "alias": "/one_store/", "yaxis": 2 }, { "alias": "/clean_locks/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_gc_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_s3_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_s3_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_s3_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_s3_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_s3_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_s3_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_s3_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_s3_gc_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_s3_gc_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_s3_gc_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_s3_gc_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_s3_gc_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote GC Duration Breakdown", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 202, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_s3_gc_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n tiflash_storage_s3_gc_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote GC Status", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, "height": null, "hideTimeOverride": false, "id": 203, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_s3_lock_mgr_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum((\n tiflash_storage_s3_lock_mgr_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Local Lock Manager status", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, "height": null, "hideTimeOverride": false, "id": 204, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_s3_lock_mgr_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_s3_lock_mgr_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Local Lock Manager QPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 }, "height": null, "hideTimeOverride": false, "id": 205, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/hit_ratio/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_fap_task_result\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_fap_task_result\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FAP result", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 }, "height": null, "hideTimeOverride": false, "id": 206, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/hit_ratio/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_fap_task_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_fap_task_state\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FAP state", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 207, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/hit_ratio/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_fap_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_fap_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_fap_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_fap_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_fap_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_fap_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_fap_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_fap_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_fap_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_fap_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_fap_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_fap_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_fap_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_fap_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FAP time by stage", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 208, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/hit_ratio/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_fap_nomatch_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_fap_nomatch_reason\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "FAP no match reason", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Disaggregated-Write", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 209, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 210, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_disaggregated_breakdown_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_disaggregated_breakdown_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_disaggregated_breakdown_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Read Duration Breakdown", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Remote Cache Operations", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 211, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "dtfile_cache_hit_ratio", "yaxis": 2 }, { "alias": "page_cache_hit_ratio", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_remote_cache\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_remote_cache\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dtfile_hit\"}[$__rate_interval])) / sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dtfile_hit|dtfile_miss\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "dtfile_cache_hit_ratio", "metric": "", "query": "(sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dtfile_hit\"}[$__rate_interval])) / sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"dtfile_hit|dtfile_miss\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"page_hit\"}[$__rate_interval])) / sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"page_hit|page_miss\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "page_cache_hit_ratio", "metric": "", "query": "(sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"page_hit\"}[$__rate_interval])) / sum(rate(tiflash_storage_remote_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"page_hit|page_miss\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Cache Operations", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Remote Cache Flow", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 212, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_remote_cache_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_remote_cache_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Cache Flow", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 213, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_remote_cache_bg_download_stage_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, stage, file_type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{stage}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_remote_cache_bg_download_stage_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, stage, file_type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, stage, file_type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{stage}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, stage, file_type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, stage, file_type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{stage}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, stage, file_type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, stage, file_type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{stage}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, stage, file_type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, stage, file_type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{stage}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, stage, file_type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (stage, file_type, $additional_groupby) / sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (stage, file_type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{stage}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (stage, file_type, $additional_groupby) / sum(rate(\n tiflash_storage_remote_cache_bg_download_stage_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (stage, file_type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Cache BG Download Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 214, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_remote_cache_wait_on_downloading_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, result, file_type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{result}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_remote_cache_wait_on_downloading_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, result, file_type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, result, file_type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{result}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, result, file_type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, result, file_type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{result}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, result, file_type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, result, file_type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{result}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, result, file_type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, result, file_type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{result}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, result, file_type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (result, file_type, $additional_groupby) / sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (result, file_type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{result}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (result, file_type, $additional_groupby) / sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (result, file_type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Cache Wait on Downloading Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 215, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_result\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (result, file_type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{result}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_result\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (result, file_type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Cache Wait on Downloading OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 216, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (result, file_type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{result}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_remote_cache_wait_on_downloading_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (result, file_type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Cache Wait on Downloading Flow", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 217, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_storage_remote_cache_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum((\n tiflash_storage_remote_cache_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type, instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Cache Gauge", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 218, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_remote_cache_reject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (reason, file_type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{reason}}-{{file_type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_remote_cache_reject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (reason, file_type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Cache Reject Download Type OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Remote Cache Usage", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 40 }, "height": null, "hideTimeOverride": false, "id": 219, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_DTFileCacheCapacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "DTFileCapacity-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_DTFileCacheCapacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_DTFileCacheUsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "DTFileUsed-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_DTFileCacheUsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_PageCacheCapacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "PageCapacity-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_PageCacheCapacity\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_PageCacheUsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "PageUsed-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_PageCacheUsed\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Remote Cache Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Memory Usage of Storage Tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 40 }, "height": null, "hideTimeOverride": false, "id": 220, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_MemoryTrackingQueryStorageTask\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "MemoryTrackingQueryStorageTask-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_MemoryTrackingQueryStorageTask\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_MemoryTrackingFetchPages\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "MemoryTrackingFetchPages-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_MemoryTrackingFetchPages\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_DT_DeltaIndexCacheSize\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "DeltaIndexCacheSize-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_DT_DeltaIndexCacheSize\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_MemoryTrackingSharedColumnData\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "SharedColumnData-{{instance}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_MemoryTrackingSharedColumnData\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Memory Usage of Storage Tasks", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "DeltaIndex cache of ReadNodes", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 48 }, "height": null, "hideTimeOverride": false, "id": 221, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/hit_ratio/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_mvcc_index_cache\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-{{instance}}", "metric": "", "query": "sum(rate(\n tiflash_storage_mvcc_index_cache\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, instance) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_storage_mvcc_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"hit\"}[$__rate_interval])) by (instance) / sum(rate(tiflash_storage_mvcc_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "hit_ratio-{{instance}}", "metric": "", "query": "(sum(rate(tiflash_storage_mvcc_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=~\"hit\"}[$__rate_interval])) by (instance) / sum(rate(tiflash_storage_mvcc_index_cache{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "MVCCIndexCache", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Duration of storage's internal sub tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 48 }, "height": null, "hideTimeOverride": false, "id": 222, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_subtask_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_subtask_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_subtask_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_subtask_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_subtask_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_subtask_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"place_index_update\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PlaceIndex Tasks Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Total number of storage's internal sub tasks", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 223, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_place_index_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_place_index_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_storage_subtask_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"place_index_update\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_storage_subtask_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\"place_index_update\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PlaceIndexTask/Reuse OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 56 }, "height": null, "hideTimeOverride": false, "id": 224, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_place_index_stats_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_place_index_stats_count_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_place_index_stats_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_place_index_stats_count_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum by (type, $additional_groupby) (rate(tiflash_storage_place_index_stats_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) / sum by (type, $additional_groupby) (rate(tiflash_storage_place_index_stats_count_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "sum by (type, $additional_groupby) (rate(tiflash_storage_place_index_stats_count_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval])) / sum by (type, $additional_groupby) (rate(tiflash_storage_place_index_stats_count_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "PlaceIndex update rows/deletes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Disaggregated-Compute", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 225, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "S3 read/write throughput", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 226, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3WriteBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "S3WriteBytes {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3WriteBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3ReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3ReadBytes {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3ReadBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3WriteDMFileBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3WriteDMFileBytes {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3WriteDMFileBytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "S3 Bytes", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "binBps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "S3 OPS", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 227, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3PutObject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "S3PutObject {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3PutObject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3GetObject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3GetObject {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3GetObject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3HeadObject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3HeadObject {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3HeadObject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3ListObjects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3ListObjects {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3ListObjects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3DeleteObject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3DeleteObject {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3DeleteObject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3CopyObject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3CopyObject {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3CopyObject\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3CreateMultipartUpload\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3CreateMultipartUpload {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3CreateMultipartUpload\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3UploadPart\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3UploadPart {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3UploadPart\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3CompleteMultipartUpload\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3CompleteMultipartUpload {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3CompleteMultipartUpload\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3PutDMFile\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3PutDMFile {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3PutDMFile\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3IORead\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3IORead {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3IORead\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3IOSeek\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3IOSeek {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3IOSeek\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "S3 OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "S3 Retry OPS", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 228, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3GetObjectRetry\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "S3GetObjectRetry {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3GetObjectRetry\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3PutObjectRetry\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3PutObjectRetry {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3PutObjectRetry\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3PutDMFileRetry\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3PutDMFileRetry {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3PutDMFileRetry\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3IOReadError\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "S3IOReadError {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3IOReadError\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3IOSeekError\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "S3IOSeekError {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3IOSeekError\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3IOSeekBackward\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "S3IOSeekBackward {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3IOSeekBackward\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "S3 Retry OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "S3 Request Duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 229, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_s3_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_s3_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_s3_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_s3_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_s3_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_s3_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_s3_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_s3_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_s3_request_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_s3_request_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_s3_request_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_s3_request_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "S3 Request Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "S3 HTTP OPS", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 230, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3ReadRequestsCount\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "read-count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3ReadRequestsCount\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3WriteRequestsCount\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "write-count {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3WriteRequestsCount\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3ReadRequestsErrors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "read-error {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3ReadRequestsErrors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3WriteRequestsErrors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "write-error {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3WriteRequestsErrors\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3ReadRequestsThrottling\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "read-throttling {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3ReadRequestsThrottling\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3WriteRequestsThrottling\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "write-throttling {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3WriteRequestsThrottling\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3ReadRequestsRedirects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "read-redirects {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3ReadRequestsRedirects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3WriteRequestsRedirects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "write-redirects {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3WriteRequestsRedirects\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3ReadRequestsNotFound\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "read-notfound {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3ReadRequestsNotFound\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3WriteRequestsNotFound\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "write-notfound {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3WriteRequestsNotFound\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "S3 HTTP OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "S3 HTTP Request Duration", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 231, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_http_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-max {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_storage_s3_http_request_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_s3_http_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-9999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_storage_s3_http_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_s3_http_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-999 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_storage_s3_http_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_s3_http_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-99 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_storage_s3_http_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_s3_http_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-80 {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_storage_s3_http_request_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, type, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_storage_s3_http_request_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_s3_http_request_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}-avg {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_storage_s3_http_request_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_storage_s3_http_request_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "S3 HTTP Request Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "S3 HTTP OPS", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 232, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_S3Requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3Requests {{$additional_groupby}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_S3Requests\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_system_current_metric_S3RandomAccessFile\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "S3RandomAccessFile {{$additional_groupby}}", "metric": "", "query": "sum((\n tiflash_system_current_metric_S3RandomAccessFile\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "S3 on-going instances", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 233, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3IOReadError\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "S3IOReadError {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3IOReadError\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3IOSeekError\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "S3IOSeekError {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3IOSeekError\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3IOSeekBackward\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "S3IOSeekBackward {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3IOSeekBackward\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3IORead\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "S3IORead {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3IORead\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_system_profile_event_S3IOSeek\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "S3IOSeek {{$additional_groupby}}", "metric": "", "query": "sum(rate(\n tiflash_system_profile_event_S3IOSeek\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "S3RandomAccessFile OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "S3", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 234, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 235, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_pipeline_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\".*_task_thread_pool_size\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "max((\n tiflash_pipeline_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\".*_task_thread_pool_size\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Thread Pool Size", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 236, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_pipeline_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\".*_tasks_count\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "max((\n tiflash_pipeline_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\".*_tasks_count\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_pipeline_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\".*_tasks_count\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "sum({{type}})", "metric": "", "query": "sum((\n tiflash_pipeline_scheduler\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=~\".*_tasks_count\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Count", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 237, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_pipeline_task_change_to_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{type}}", "metric": "", "query": "sum(rate(\n tiflash_pipeline_task_change_to_status\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Status Change OPS", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 238, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_pipeline_task_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_pipeline_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_pipeline_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_pipeline_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_pipeline_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_pipeline_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_pipeline_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_pipeline_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{type}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_pipeline_task_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_pipeline_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_pipeline_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{type}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_pipeline_task_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) / sum(rate(\n tiflash_pipeline_task_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, $additional_groupby) )", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_execute\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_execute\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-cpu_execute", "metric": "", "query": "(sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_execute\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_execute\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_queue\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_queue\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-cpu_queue", "metric": "", "query": "(sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_queue\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu_queue\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_execute\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_execute\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-io_execute", "metric": "", "query": "(sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_execute\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_execute\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_queue\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_queue\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-io_queue", "metric": "", "query": "(sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_queue\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io_queue\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"await\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"await\"}[$__rate_interval])))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-await", "metric": "", "query": "(sum(rate(tiflash_pipeline_task_duration_seconds_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"await\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"await\"}[$__rate_interval])))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 239, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.95,(\n sum(rate(\n tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "95-{{type}}", "metric": "", "query": "histogram_quantile(0.95,(\n sum(rate(\n tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.95,(\n sum(rate(\n tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{type}}", "metric": "", "query": "histogram_quantile(0.95,(\n sum(rate(\n tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{type}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (type, le) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type) / 1000000000)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "100-{{type}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, type) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu\"}[$__rate_interval]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-cpu", "metric": "", "query": "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"cpu\"}[$__rate_interval]))", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io\"}[$__rate_interval]))", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-io", "metric": "", "query": "sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_sum{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io\"}[$__rate_interval])) / sum(rate(tiflash_pipeline_task_execute_max_time_seconds_per_round_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"io\"}[$__rate_interval]))", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Task Max Execute Time Per Round", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 16 }, "height": null, "hideTimeOverride": false, "id": 240, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"cpu_pool\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"cpu_pool\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"cpu_pool\", instance=~\"$tiflash_role\"}) by (instance)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"cpu_pool\", instance=~\"$tiflash_role\"}) by (instance)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads CPU of CPU Task Thread Pool", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 241, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"io_pool\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"io_pool\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"io_pool\", instance=~\"$tiflash_role\"}) by (instance)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"io_pool\", instance=~\"$tiflash_role\"}) by (instance)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads CPU of IO Task Thread Pool", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 24 }, "height": null, "hideTimeOverride": false, "id": 242, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "Limit", "color": "#F2495C", "hideTooltip": true, "legend": false, "linewidth": 2, "nullPointMode": "connected" } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"WaitReactor\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{name}} {{instance}}", "metric": "", "query": "sum(rate(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"WaitReactor\", instance=~\"$tiflash_role\"}[$__rate_interval])) by (instance)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "count(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"WaitReactor\", instance=~\"$tiflash_role\"}) by (instance)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "Limit", "metric": "", "query": "count(tiflash_proxy_thread_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", name=~\"WaitReactor\", instance=~\"$tiflash_role\"}) by (instance)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Threads CPU of Wait Reactor", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "wait notify task details", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 32 }, "height": null, "hideTimeOverride": false, "id": 243, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_pipeline_wait_on_notify_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "max((\n tiflash_pipeline_wait_on_notify_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (instance, type) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum((\n tiflash_pipeline_wait_on_notify_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "sum({{type}})", "metric": "", "query": "sum((\n tiflash_pipeline_wait_on_notify_tasks\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) by (type) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Wait notify task details", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Pipeline Model", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 244, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "Metas of resource group", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 245, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": true, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_resource_group\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"remaining_tokens\"}\n \n)) by (instance, resource_group) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "remaining_tokens-{{instance}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_resource_group\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"remaining_tokens\"}\n \n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_resource_group\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"avg_speed\"}\n \n)) by (instance, resource_group) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg_speed-{{instance}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_resource_group\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"avg_speed\"}\n \n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_resource_group_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"total_consumption\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "total_consumption-{{instance}}-{{resource_group}}", "metric": "", "query": "sum(rate(\n tiflash_resource_group_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"total_consumption\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_resource_group\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"bucket_fill_rate\"}\n \n)) by (instance, resource_group) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "bucket_fill_rate-{{instance}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_resource_group\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"bucket_fill_rate\"}\n \n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "max((\n tiflash_resource_group\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"bucket_capacity\"}\n \n)) by (instance, resource_group) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "bucket_capacity-{{instance}}-{{resource_group}}", "metric": "", "query": "max((\n tiflash_resource_group\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"bucket_capacity\"}\n \n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_resource_group_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"request_gac_count\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "request_gac_count-{{instance}}-{{resource_group}}", "metric": "", "query": "sum(rate(\n tiflash_resource_group_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"request_gac_count\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_resource_group_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"gac_req_ru_consumption_delta\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "gac_req_ru_consumption_delta-{{instance}}-{{resource_group}}", "metric": "", "query": "sum(rate(\n tiflash_resource_group_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"gac_req_ru_consumption_delta\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_resource_group_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"compute_ru_consumption\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "compute_ru_consumption-{{instance}}-{{resource_group}}", "metric": "", "query": "sum(rate(\n tiflash_resource_group_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"compute_ru_consumption\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(\n tiflash_resource_group_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"storage_ru_consumption\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "storage_ru_consumption-{{instance}}-{{resource_group}}", "metric": "", "query": "sum(rate(\n tiflash_resource_group_counter\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"storage_ru_consumption\"}\n [$__rate_interval]\n)) by (instance, resource_group) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "TiFlash Resource Group", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "Request Unit for tidb-serverless charging", "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 246, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/sum/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_storage_sync_replica_ru{instance=~\"$tiflash_role\"}[$__rate_interval])) by (keyspace_id, $additional_groupby)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "replica-sync-rate-{{keyspace_id}}", "metric": "", "query": "sum(rate(tiflash_storage_sync_replica_ru{instance=~\"$tiflash_role\"}[$__rate_interval])) by (keyspace_id, $additional_groupby)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(tiflash_storage_sync_replica_ru{instance=~\"$tiflash_role\"}[24h])) by (keyspace_id, $additional_groupby)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "replica-sync-sum-24h-{{keyspace_id}} {{$additional_groupby}}", "metric": "", "query": "sum(increase(tiflash_storage_sync_replica_ru{instance=~\"$tiflash_role\"}[24h])) by (keyspace_id, $additional_groupby)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_compute_request_unit{instance=~\"$tiflash_role\"}[$__rate_interval])) by (cluster_id, $additional_groupby)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "query-rate-{{cluster_id}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(tiflash_compute_request_unit{instance=~\"$tiflash_role\"}[$__rate_interval])) by (cluster_id, $additional_groupby)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(increase(tiflash_compute_request_unit{instance=~\"$tiflash_role\"}[24h])) by (cluster_id, $additional_groupby)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "query-sum-24h-{{cluster_id}} {{$additional_groupby}}", "metric": "", "query": "sum(increase(tiflash_compute_request_unit{instance=~\"$tiflash_role\"}[24h])) by (cluster_id, $additional_groupby)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate(tiflash_storage_ru_read_bytes{instance=~\"$tiflash_role\"}[$__rate_interval])) by (keyspace, resource_group, type, $additional_groupby) / (64 * 1024)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "storage-{{keyspace}}_{{resource_group}}_{{type}} {{$additional_groupby}}", "metric": "", "query": "sum(rate(tiflash_storage_ru_read_bytes{instance=~\"$tiflash_role\"}[$__rate_interval])) by (keyspace, resource_group, type, $additional_groupby) / (64 * 1024)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Request Unit", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "cps", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "TiFlash Resource Control", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 247, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 248, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, path, $additional_groupby) / 1000000000)", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "max-{{path}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(1.00, sum(round(1000000000*rate(tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}[$__rate_interval]))) by (le, path, $additional_groupby) / 1000000000)", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, path, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "9999-{{path}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.9999,(\n sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, path, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, path, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "999-{{path}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.999,(\n sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, path, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, path, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "99-{{path}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.99,(\n sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, path, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, path, $additional_groupby) \n \n \n)) ", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "80-{{path}} {{$additional_groupby}}", "metric": "", "query": "histogram_quantile(0.80,(\n sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_bucket\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (le, path, $additional_groupby) \n \n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "(sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) / sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) )", "format": "time_series", "hide": true, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "avg-{{path}} {{$additional_groupby}}", "metric": "", "query": "(sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_sum\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) / sum(rate(\n tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_count\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$proxy_instance\",instance=~\"$tiflash_role\"}\n [$__rate_interval]\n)) by (path, $additional_groupby) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Status API Request Duration", "tooltip": { "msResolution": true, "shared": true, "sort": 2, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 1, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 249, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum(rate( tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"} [$__rate_interval] )) by (path, $additional_groupby)", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{path}} {{$additional_groupby}}", "metric": "", "query": "sum(rate( tiflash_proxy_tikv_status_server_proxy_request_duration_seconds_count {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\", instance=~\"$proxy_instance\", instance=~\"$tiflash_role\"} [$__rate_interval] )) by (path, $additional_groupby)", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Status API Request (op/s)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "ops", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Status Server", "transformations": [], "transparent": false, "type": "row" }, { "cacheTimeout": null, "collapsed": true, "datasource": null, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] } } }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 250, "interval": null, "links": [], "maxDataPoints": 100, "maxPerRow": null, "minSpan": null, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 0, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 251, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "sum by (type, instance) ( tiflash_vector_index_active_instances{ k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\" } )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "sum by (type, instance) ( tiflash_vector_index_active_instances{ k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\" } )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "In-Memory Vector Index Instances", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 0, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 }, "height": null, "hideTimeOverride": false, "id": 252, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_vector_index_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{instance}}-{{type}}", "metric": "", "query": "((\n tiflash_vector_index_memory_usage\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\"}\n \n)) ", "refId": "", "step": 10, "target": "" }, { "datasource": "${DS_TEST-CLUSTER}", "expr": "((\n tiflash_process_rss_by_type_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"file\"}\n \n)) ", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 2, "legendFormat": "{{instance}}-RssFile", "metric": "", "query": "((\n tiflash_process_rss_by_type_bytes\n {k8s_cluster=\"$k8s_cluster\",tidb_cluster=\"$tidb_cluster\",instance=~\"$instance\",instance=~\"$tiflash_role\",type=\"file\"}\n \n)) ", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "Vector Index Estimated Memory Usage", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 253, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [ { "alias": "/download/", "yaxis": 2 } ], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile( 0.999, sum(rate( tiflash_vector_index_duration_bucket{ k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type!=\"build\" } [$__rate_interval] )) by (le, type) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "histogram_quantile( 0.999, sum(rate( tiflash_vector_index_duration_bucket{ k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type!=\"build\" } [$__rate_interval] )) by (le, type) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.9% Vector Search Duration (Per Request)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true } ], "yaxis": { "align": false, "alignLevel": 0 } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": null, "editable": true, "error": false, "fieldConfig": { "defaults": { "thresholds": { "mode": "absolute", "steps": [] }, "unit": "" } }, "fill": 0, "grid": { "threshold1": null, "threshold1Color": "rgba(216, 200, 27, 0.27)", "threshold2": null, "threshold2Color": "rgba(234, 112, 112, 0.22)" }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 }, "height": null, "hideTimeOverride": false, "id": 254, "interval": null, "isNew": true, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideEmpty": true, "hideZero": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": null, "maxPerRow": null, "minSpan": null, "nullPointMode": "null as zero", "options": { "alertThreshold": true, "dataLinks": [] }, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeat": null, "repeatDirection": null, "seriesOverrides": [], "span": null, "stack": false, "steppedLine": false, "targets": [ { "datasource": "${DS_TEST-CLUSTER}", "expr": "histogram_quantile( 0.999, sum(rate( tiflash_vector_index_duration_bucket{ k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"build\" } [$__rate_interval] )) by (le, type) )", "format": "time_series", "hide": false, "instant": false, "interval": "", "intervalFactor": 1, "legendFormat": "{{type}}", "metric": "", "query": "histogram_quantile( 0.999, sum(rate( tiflash_vector_index_duration_bucket{ k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", instance=~\"$tiflash_role\", type=\"build\" } [$__rate_interval] )) by (le, type) )", "refId": "", "step": 10, "target": "" } ], "thresholds": [], "timeFrom": null, "timeShift": null, "title": "99.9% Vector Index Build Duration (Per DMFile Column)", "tooltip": { "msResolution": true, "shared": true, "sort": 0, "value_type": "individual" }, "transformations": [], "transparent": false, "type": "graph", "xaxis": { "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": 0 } } ], "repeat": null, "repeatDirection": null, "span": null, "targets": [], "timeFrom": null, "timeShift": null, "title": "Vector Search", "transformations": [], "transparent": false, "type": "row" } ], "refresh": "1m", "rows": [], "schemaVersion": 14, "sharedCrosshair": false, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "K8s-cluster", "multi": false, "name": "k8s_cluster", "options": [], "query": "label_values(tiflash_system_profile_event_Query, k8s_cluster)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", "options": [], "query": "label_values(tiflash_system_profile_event_Query{k8s_cluster=\"$k8s_cluster\"}, tidb_cluster)", "refresh": 2, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "Instance", "multi": true, "name": "instance", "options": [], "query": "label_values(tiflash_system_profile_event_Query{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", "refresh": 1, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "tags": [], "text": null, "value": null }, "datasource": "${DS_TEST-CLUSTER}", "hide": 0, "includeAll": true, "label": "Proxy Instance", "multi": true, "name": "proxy_instance", "options": [], "query": "label_values(tiflash_proxy_process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}, instance)", "refresh": 1, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "query", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": true, "text": "none", "value": "none" }, "datasource": null, "hide": 0, "includeAll": false, "label": "additional_groupby", "multi": false, "name": "additional_groupby", "options": [ { "selected": true, "text": "none", "value": "none" }, { "selected": false, "text": "instance", "value": "instance" } ], "query": "none,instance", "refresh": 1, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "custom", "useTags": false }, { "allValue": null, "auto": false, "auto_count": 30, "auto_min": "10s", "current": { "selected": true, "text": "All", "value": ".*" }, "datasource": null, "hide": 0, "includeAll": false, "label": "Role", "multi": false, "name": "tiflash_role", "options": [ { "selected": true, "text": "All", "value": ".*" }, { "selected": false, "text": "Write", "value": ".*write-tiflash.*" }, { "selected": false, "text": "Compute", "value": ".*compute-tiflash.*" } ], "query": "All : .*, Write : .*write-tiflash.*, Compute : .*compute-tiflash.*", "refresh": 1, "regex": null, "sort": 1, "tagValuesQuery": null, "tagsQuery": null, "type": "custom", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "hidden": false, "nowDelay": null, "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "[SharedScope]Test-Cluster-TiFlash-Summary", "uid": "SVbh2xUWk", "version": 0 }