{ "__inputs": [ { "name": "DS_TEST-CLUSTER", "label": "Prometheus", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "6.0.1" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "5.0.0" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "5.0.0" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "${DS_TEST-CLUSTER}", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 1, "id": null, "iteration": 1606814402924, "links": [], "panels": [ { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 11269, "panels": [], "title": "Summary", "type": "row" }, { "aliasColors": { "copr-scan-number": "light-red", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green", "write-rpc (total)": "green", "write-rpc-total": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 4, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 1 }, "id": 9877, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "/(.*)-write/", "fill": 0, "legend": false, "lines": true, "linewidth": 1, "yaxis": 2, "zindex": 3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m])) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{instance}}-write", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc (total)", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Imbalanced write ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "copr-scan-number": "light-red", "kv_commit": "green", "read-rpc": "rgb(123, 187, 244)", "read-rpc (total)": "rgb(123, 187, 244)", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 4, "grid": {}, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 1 }, "id": 10029, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 0, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "/(.*)-read/", "fill": 0, "legend": false, "linewidth": 1, "yaxis": 2, "zindex": 3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_get|kv_scan|kv_batch_get|kv_batch_get_command|coprocessor\"}[1m])) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{instance}}-read", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_get|kv_scan|kv_batch_get|kv_batch_get_command|coprocessor\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "read-rpc (total)", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Imbalanced read ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "copr-duration-99%": "dark-orange", "copr-duration-999%": "light-orange", "copr-duration-max": "light-orange", "copr-rpc-qps": "rgb(117, 99, 224)", "get-rpc-qps": "semi-dark-red", "kv_commit": "green", "kv_pessimistic_lock": "rgba(255, 29, 6, 0.84)", "kv_prewrite": "rgb(11, 38, 239)", "neg-raftdb-wal-duration": "rgb(226, 49, 85)", "raftdb-wal-duration": "semi-dark-red", "read-rpc-duration-99%": "semi-dark-orange", "read-rpc-duration-999%": "dark-yellow", "write-rpc": "green", "write-rpc-99%": "semi-dark-red", "write-rpc-999%": "semi-dark-red", "write-rpc-duration": "semi-dark-red", "write-rpc-qps": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 8 }, "id": 10638, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "get-rpc-qps", "yaxis": 2 }, { "alias": "write-rpc-qps", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_batch_get|kv_batch_get_command|kv_get|kv_scan\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "intervalFactor": 1, "legendFormat": "get-rpc-qps", "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "write-rpc-qps", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "BatchGet-RPC & Get-RPC QPS Follow Write-RPC ?", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "cop-rpc-qps": "rgb(255, 1, 243)", "copr-duration-99%": "dark-orange", "copr-duration-999%": "light-orange", "copr-duration-max": "light-orange", "copr-rpc-qps": "rgb(117, 99, 224)", "get-rpc-qps": "semi-dark-red", "kv_commit": "green", "kv_pessimistic_lock": "rgba(255, 29, 6, 0.84)", "kv_prewrite": "rgb(11, 38, 239)", "neg-raftdb-wal-duration": "rgb(226, 49, 85)", "raftdb-wal-duration": "semi-dark-red", "read-rpc-duration-99%": "semi-dark-orange", "read-rpc-duration-999%": "dark-yellow", "write-rpc": "green", "write-rpc-99%": "semi-dark-red", "write-rpc-999%": "semi-dark-red", "write-rpc-duration": "semi-dark-red", "write-rpc-qps": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 8 }, "id": 11279, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "get-rpc-qps", "yaxis": 2 }, { "alias": "cop-rpc-qps", "yaxis": 2 }, { "alias": "write-rpc-qps", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"coprocessor\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "intervalFactor": 1, "legendFormat": "cop-rpc-qps", "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "write-rpc-qps", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Coprocessor-RPC QPS Follows Write-RPC ?", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "172.16.4.231:16812-lat-999%": "rgb(255, 138, 140)", "172.16.4.55:23581-lat-999%": "light-red", "172.16.5.113:16810-lat-999%": "rgb(54, 51, 209)", "172.16.5.34:23581-lat-999%": "dark-red", "copr-scan-number": "light-red", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green", "write-rpc (total)": "green", "write-rpc-total": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 13 }, "id": 11271, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "/(.*)-lat-999%/", "legend": false, "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m])) by (le, instance))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{instance}}-lat-999%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc (total)", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Some instances write too slow ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "172.16.4.231:16812-lat-999%": "rgb(255, 138, 140)", "172.16.4.55:23581-lat-999%": "super-light-blue", "172.16.5.113:16810-lat-999%": "rgb(54, 51, 209)", "172.16.5.33:16808-lat-999%": "super-light-red", "172.16.5.34:23581-lat-999%": "dark-purple", "copr-scan-number": "light-red", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green", "write-rpc (total)": "green", "write-rpc-total": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 13 }, "id": 11272, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "/(.*)-lat-999%/", "legend": false, "transform": "negative-Y", "yaxis": 2, "zindex": 3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_get|kv_scan|kv_batch_get|kv_batch_get_command|coprocessor\"}[1m])) by (le, instance))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{instance}}-lat-999%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc (total)", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Some instances read too slow (Get + Coprocessor)?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "kv_commit": "green", "kv_pessimistic_lock": "rgba(255, 29, 6, 0.84)", "kv_prewrite": "rgb(11, 38, 239)", "neg-raftdb-wal-duration": "rgb(226, 49, 85)", "raftdb-wal-duration": "semi-dark-red", "write-rpc": "green", "write-rpc-duration": "semi-dark-red", "write-rpc-duration-99%": "super-light-blue", "write-rpc-duration-999%": "semi-dark-blue", "write-rpc-duration-max": "super-light-blue" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 18 }, "id": 8637, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "write-rpc-duration-999%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc-duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc-duration-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "intervalFactor": 1, "legendFormat": "write-rpc", "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.999, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "write-rpc-duration-999%", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "write-rpc-duration-99%", "refId": "C" }, { "expr": "histogram_quantile(1, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "write-rpc-duration-max", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Write-RPC too slow ?", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "kv_commit": "green", "kv_pessimistic_lock": "rgba(255, 29, 6, 0.84)", "kv_prewrite": "rgb(11, 38, 239)", "neg-raftdb-wal-duration": "rgb(226, 49, 85)", "raftdb-wal-duration": "semi-dark-red", "read-rpc-duration-99%": "semi-dark-orange", "read-rpc-duration-999%": "light-orange", "read-rpc-duration-max": "light-orange", "write-rpc": "green", "write-rpc-99%": "semi-dark-red", "write-rpc-999%": "semi-dark-red", "write-rpc-duration": "semi-dark-red" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 18 }, "id": 9254, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "read-rpc-duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "read-rpc-duration-999%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "read-rpc-duration-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "intervalFactor": 1, "legendFormat": "write-rpc", "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.999, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_get|kv_scan|kv_batch_get|kv_batch_get_command|coprocessor\"}[1m])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "read-rpc-duration-999%", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_get|kv_batch_get|kv_batch_get_command|kv_scan|coprocessor\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "read-rpc-duration-99%", "refId": "C" }, { "expr": "histogram_quantile(0.999, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_get|kv_scan|kv_batch_get|kv_batch_get_command|coprocessor\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "read-rpc-duration-max", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Read-RPC too slow ? (Get + Coprocessor)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "duration-99": "dark-red", "duration-99%": "blue", "duration-999%": "yellow", "duration-max": "rgb(242, 4, 7)", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 23 }, "id": 9875, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "duration-max", "transform": "negative-Y", "yaxis": 2 }, { "alias": "duration-99", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "max(tikv_engine_write_stall{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_max\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "duration-max", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "max(tikv_engine_write_stall{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=\"write_stall_percentile99\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "duration-99", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Has write stall ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "µs", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "copr-duration-99%": "dark-orange", "copr-duration-999%": "light-orange", "copr-duration-max": "light-orange", "copr-rpc-qps": "rgb(117, 99, 224)", "kv_commit": "green", "kv_pessimistic_lock": "rgba(255, 29, 6, 0.84)", "kv_prewrite": "rgb(11, 38, 239)", "neg-raftdb-wal-duration": "rgb(226, 49, 85)", "raftdb-wal-duration": "semi-dark-red", "read-rpc-duration-99%": "semi-dark-orange", "read-rpc-duration-999%": "dark-yellow", "write-rpc": "green", "write-rpc-99%": "semi-dark-red", "write-rpc-999%": "semi-dark-red", "write-rpc-duration": "semi-dark-red" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 23 }, "id": 11260, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "copr-duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "copr-duration-999%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "copr-duration-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"coprocessor\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "intervalFactor": 1, "legendFormat": "copr-rpc-qps", "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.999, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"coprocessor\"}[1m])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "copr-duration-999%", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"coprocessor\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "copr-duration-99%", "refId": "C" }, { "expr": "histogram_quantile(1, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"coprocessor\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "copr-duration-max", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Coprocessor-RPC too slow ?", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "duration-99%": "purple", "duration-999%": "rgb(9, 224, 117)", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 28 }, "id": 11008, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "duration-999%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "duration-max", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"prewrite|commit|acquire_pessimistic_lock\"}[1m])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "duration-999%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "histogram_quantile(1, sum(rate(tikv_scheduler_latch_wait_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"prewrite|commit|acquire_pessimistic_lock\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "duration-max", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Latch wait too long (incorrectly included scheduler-wait-duration now) ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "copr-duration-99%": "dark-orange", "copr-duration-999%": "light-orange", "copr-duration-max": "light-orange", "copr-rpc-qps": "rgb(117, 99, 224)", "get-rpc-qps": "light-red", "kv_commit": "green", "kv_pessimistic_lock": "rgba(255, 29, 6, 0.84)", "kv_prewrite": "rgb(11, 38, 239)", "neg-raftdb-wal-duration": "rgb(226, 49, 85)", "raftdb-wal-duration": "semi-dark-red", "read-rpc-duration-99%": "semi-dark-orange", "read-rpc-duration-999%": "dark-yellow", "write-rpc": "green", "write-rpc-99%": "semi-dark-red", "write-rpc-999%": "semi-dark-red", "write-rpc-duration": "semi-dark-red" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 28 }, "id": 11278, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "get-duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "get-duration-999%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "get-duration-max", "transform": "negative-Y", "yaxis": 2 }, { "alias": "get-rpc-qps", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_batch_get|kv_batch_get_command|kv_get|kv_scan\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "intervalFactor": 1, "legendFormat": "get-rpc-qps", "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 }, { "expr": "histogram_quantile(0.999, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_batch_get|kv_batch_get_command|kv_get|kv_scan\"}[1m])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "get-duration-999%", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_batch_get|kv_batch_get_command|kv_get|kv_scan\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "get-duration-99%", "refId": "C" }, { "expr": "histogram_quantile(1, sum(rate(tikv_grpc_msg_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_batch_get|kv_batch_get_command|kv_get|kv_scan\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "get-duration-max", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "BatchGet-RPC | Get-RPC too slow ?", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "duration-99%": "blue", "duration-999%": "yellow", "kv_commit": "green", "pending-bytes": "orange", "pending-bytes-kv": "rgb(21, 121, 231)", "pending-bytes-raft": "dark-red", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 33 }, "id": 9876, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "pending-bytes-kv", "transform": "negative-Y", "yaxis": 2 }, { "alias": "pending-bytes-raft", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(tikv_engine_pending_compaction_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "pending-bytes-kv", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "sum(tikv_engine_pending_compaction_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "pending-bytes-raft", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Too many compaction pending bytes accumulating ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "decbytes", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "172.16.4.55:23581-pending-bytes-KVDB": "rgb(100, 144, 211)", "duration-99%": "blue", "duration-999%": "yellow", "kv_commit": "green", "pending-bytes": "orange", "pending-bytes-kv": "rgb(21, 121, 231)", "pending-bytes-raft": "dark-red", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 33 }, "id": 11282, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "/(.*)-pending-bytes-(.*)/", "legend": false, "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(tikv_engine_pending_compaction_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\"}) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{instance}}-pending-bytes-KVDB", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Some instances' KVDB compaction pending bytes accumulating ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "decbytes", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "kv_commit": "green", "move-peer-opm": "red", "move-peer-ops": "dark-red", "pd-worker-pending-task": "dark-red", "wait-duration-99%": "rgb(39, 22, 196)", "wait-duration-999%": "super-light-red", "wait-max": "semi-dark-red", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 38 }, "id": 11276, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "move-peer-opm", "transform": "negative-Y", "yaxis": 2 }, { "alias": "transfer-leader-opm", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(delta(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"finish\", type=~\"balance-region|move-hot-write-peer|move-hot-read-peer|make-up-replica\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "move-peer-opm", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "sum(delta(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"finish\", type=~\"transfer-hot-read-leader|transfer-hot-write-leader|balance-leader\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "transfer-leader-opm", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "PD has too many scheduling operators ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "opm", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "kv_commit": "green", "move-peer-opm": "red", "move-peer-ops": "dark-red", "pd-worker-pending-task": "dark-red", "region-balancing-opm": "dark-red", "region-balancing-ops": "dark-red", "wait-duration-99%": "rgb(39, 22, 196)", "wait-duration-999%": "super-light-red", "wait-max": "semi-dark-red", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 38 }, "id": 11280, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "/region-balancing-op(.*)/", "transform": "negative-Y", "yaxis": 2 }, { "alias": "/transfer-leader-op(.*)/", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(delta(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"finish\", type=~\"balance-region\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "region-balancing-opm", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "PD region balancing caused jitter ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "opm", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "kv_commit": "green", "pd-worker-pending-task": "dark-red", "wait-duration-99%": "rgb(39, 22, 196)", "wait-duration-999%": "super-light-red", "wait-max": "semi-dark-red", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 43 }, "id": 11275, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "pd-worker-pending-task", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_worker_pending_task_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=\"pd-worker\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "pd-worker-pending-task", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "PD client has a lot pending tasks ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "balance-region-ops": "dark-red", "kv_commit": "green", "move-peer-opm": "red", "pd-worker-pending-task": "dark-red", "region-balancing-ops": "dark-red", "wait-duration-99%": "rgb(39, 22, 196)", "wait-duration-999%": "super-light-red", "wait-max": "semi-dark-red", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 1, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 43 }, "id": 11277, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "region-balancing-opm", "fill": 10, "transform": "negative-Y", "yaxis": 2 }, { "alias": "/store-(.*)-used/", "legend": false, "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(delta(pd_schedule_operators_count{tidb_cluster=\"$tidb_cluster\", event=\"finish\", type=~\"balance-region\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "region-balancing-opm", "refId": "D" }, { "expr": "1 - sum(pd_scheduler_store_status{tidb_cluster=\"$tidb_cluster\", type=\"store_available\"}) by (store) / sum(pd_scheduler_store_status{tidb_cluster=\"$tidb_cluster\", type=\"store_capacity\"}) by (store)", "format": "time_series", "instant": false, "intervalFactor": 1, "legendFormat": "store-{{store}}-used", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "PD region balancing reason", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "opm", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "172.16.4.39:9239-cpu-usage": "dark-purple", "duration-99%": "purple", "duration-999%": "rgb(9, 224, 117)", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 48 }, "id": 11283, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "/(.*)-cpu-usage/", "legend": false, "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "100 - avg by (instance) (irate(node_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", mode=\"idle\"}[1m]) ) * 100", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{instance}}-cpu-usage", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Some instances' CPU usage have jitter ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "percent", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 53 }, "id": 11267, "panels": [], "title": "TiKV-Write affected TiDB-Write ?", "type": "row" }, { "aliasColors": { "kv_commit": "green", "wait-queue": "dark-purple", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 54 }, "id": 9255, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "wait-queue", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_futurepool_pending_task_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"sched-worker-pool\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "wait-queue", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Wait for available scheduler threads ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "kv_commit": "green", "wait-duration": "semi-dark-red", "wait-duration-99%": "dark-red", "wait-duration-999%": "dark-purple", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 54 }, "id": 8947, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "wait-duration-999%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "wait-duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "wait-duration-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(tikv_scheduler_wait_for_process_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"prewrite|commit|acquire_pessimistic_lock\"}[1m])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "wait-duration-999%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_scheduler_wait_for_process_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"prewrite|commit|acquire_pessimistic_lock\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "wait-duration-99%", "refId": "B" }, { "expr": "histogram_quantile(1, sum(rate(tikv_scheduler_wait_for_process_duration_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"prewrite|commit|acquire_pessimistic_lock\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "wait-duration-max", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Wait for available scheduler threads (no use now) ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "kv_commit": "green", "wait-duration-99%": "rgb(39, 22, 196)", "wait-duration-999%": "super-light-red", "wait-max": "semi-dark-red", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 59 }, "id": 9407, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "wait-duration-999%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "wait-duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "wait-max", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "wait-duration-999%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "wait-duration-99%", "refId": "B" }, { "expr": "histogram_quantile(1, sum(rate(tikv_raftstore_request_wait_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "wait-max", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Wait for available raftstore threads ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "kv_commit": "green", "wait-duration-99%": "rgb(39, 22, 196)", "wait-duration-999%": "super-light-red", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 59 }, "id": 9408, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "wait-duration-999%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "wait-duration-99%", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "wait-duration-999%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_raftstore_apply_wait_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "wait-duration-99%", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Wait for available apply threads ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "duration-99%": "rgb(1, 255, 180)", "duration-999%": "yellow", "duration-max": "light-red", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 64 }, "id": 10486, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "duration-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_write_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"write_percentile99\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "duration-99%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "avg(tikv_engine_write_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"write_max\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "duration-max", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "RaftDB write too slow ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "µs", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "duration-99%": "rgb(1, 255, 180)", "duration-999%": "yellow", "duration-max": "light-red", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 64 }, "id": 11258, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "duration-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_write_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"write_percentile99\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "duration-99%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "avg(tikv_engine_write_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"write_max\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "duration-max", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB write too slow ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "µs", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "compaction": "green", "flow": "rgb(1, 120, 152)", "kv_commit": "green", "read-flow": "light-red", "total": "orange", "write-flow": "rgb(0, 202, 255)", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 69 }, "id": 8945, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "read-flow", "stack": true, "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-flow", "stack": true, "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"flush_write_bytes\"}[1m])) + sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"bytes_written\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-flow", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "K" }, { "expr": "sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"bytes_read\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "read-flow", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB compaction flow has jitter? (read-flow may hit page-cache)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "compaction": "green", "flow": "rgb(1, 120, 152)", "kv_commit": "green", "read-flow": "light-red", "total": "orange", "write-flow": "rgb(0, 202, 255)", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 69 }, "id": 8946, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "read-flow", "stack": true, "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-flow", "stack": true, "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"flush_write_bytes\"}[1m])) + sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"bytes_written\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-flow", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "K" }, { "expr": "sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"bytes_read\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "read-flow", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "RaftDB compaction flow has jitter ? (read-flow may hit page-cache)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "Bps", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "172.16.4.55:23581-compaction-flow": "semi-dark-yellow", "compaction": "green", "flow": "rgb(1, 120, 152)", "kv_commit": "green", "read-flow": "light-red", "total": "orange", "write-flow": "rgb(239, 232, 103)", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 74 }, "id": 11281, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "/(.*)-flow/", "legend": false, "transform": "negative-Y", "yaxis": 2, "zindex": 3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"flush_write_bytes\"}[1m])) by (instance) + sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"bytes_written\"}[1m])) by (instance) + sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"bytes_read\"}[1m])) by(instance)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{instance}}-compaction-flow", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "K" }, { "expr": "sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"bytes_read\"}[1m]))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "read-flow", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Some instances' KVDB compaction flow has jitter ?", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "compaction": "green", "flow": "rgb(229, 12, 37)", "grpc-kv_commit": "green", "kv_commit": "green", "read-flow": "light-red", "rocksdb-cpu-172.16.4.231:16812": "super-light-orange", "rocksdb-cpu-172.16.4.232:16808": "dark-red", "rocksdb-cpu-172.16.4.55:23581": "light-orange", "rocksdb-cpu-usage-total": "light-red", "total": "orange", "total-flow": "semi-dark-red", "write-flow": "rgb(0, 202, 255)", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 74 }, "id": 11270, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "/rocksdb-cpu-(.*)/", "legend": false, "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_thread_cpu_seconds_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"rocksdb.*\"}[1m])) by (instance)", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "rocksdb-cpu-{{instance}}", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "K" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "RocksDB CPU usage has jitter ?", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "percentunit", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "batch-size-99%": "rgb(250, 0, 66)", "duration-99%": "semi-dark-red", "duration-999%": "yellow", "duration-max": "light-blue", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 79 }, "id": 11261, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "batch-size-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "batch-size-max", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_bytes_per_write{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"bytes_per_write_percentile99\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "batch-size-99%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "avg(tikv_engine_bytes_per_write{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"bytes_per_write_max\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "batch-size-max", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "RaftDB write batch too big ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "decbytes", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "batch-size-99%": "rgb(250, 0, 66)", "duration-99%": "semi-dark-red", "duration-999%": "yellow", "duration-max": "light-purple", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 79 }, "id": 11262, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "batch-size-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "batch-size-max", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_bytes_per_write{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"bytes_per_write_percentile99\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "batch-size-99%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "avg(tikv_engine_bytes_per_write{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"bytes_per_write_max\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "batch-size-max", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB write batch too big ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "decbytes", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "db_mutex_lock_nanos": "dark-red", "db_mutex_lock_nanos-999%": "dark-red", "db_mutex_lock_nanos-max": "dark-red", "duration-99%": "dark-orange", "duration-999%": "yellow", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green", "write_thread_wait": "dark-orange", "write_thread_wait-999%": "light-purple", "write_thread_wait-max": "light-purple" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 84 }, "id": 11263, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "db_mutex_lock_nanos-999", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write_thread_wait-999", "transform": "negative-Y", "yaxis": 2 }, { "alias": "db_mutex_lock_nanos-max", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write_thread_wait-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_store_perf_context_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"db_mutex_lock_nanos|write_thread_wait\"}[1m])) by (le, type))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{type}}-999", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "histogram_quantile(1, sum(rate(tikv_raftstore_store_perf_context_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"db_mutex_lock_nanos|write_thread_wait\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}}-max", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "RaftDB mutex lock too long ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "db_mutex_lock_nanos": "dark-red", "db_mutex_lock_nanos-999%": "dark-red", "db_mutex_lock_nanos-max": "dark-red", "duration-99%": "dark-orange", "duration-999%": "yellow", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green", "write_thread_wait": "dark-orange", "write_thread_wait-999%": "light-purple", "write_thread_wait-max": "light-purple" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 84 }, "id": 9571, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "db_mutex_lock_nanos-999", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write_thread_wait-999", "transform": "negative-Y", "yaxis": 2 }, { "alias": "db_mutex_lock_nanos-max", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write_thread_wait-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.999, sum(rate(tikv_raftstore_apply_perf_context_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"db_mutex_lock_nanos|write_thread_wait\"}[1m])) by (le, type))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "{{type}}-999", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "histogram_quantile(1, sum(rate(tikv_raftstore_apply_perf_context_time_duration_secs_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"db_mutex_lock_nanos|write_thread_wait\"}[1m])) by (le, type))", "format": "time_series", "intervalFactor": 1, "legendFormat": "{{type}}-max", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB mutex lock too long ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "compaction": "green", "flow": "rgb(229, 12, 37)", "grpc-kv_commit": "green", "kv_commit": "green", "read-flow": "light-red", "total": "orange", "total-flow": "semi-dark-red", "write-flow": "rgb(0, 202, 255)", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 89 }, "id": 9099, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "write-flow", "stack": true, "transform": "negative-Y", "yaxis": 2 }, { "alias": "read-flow", "stack": true, "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"wal_file_bytes\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-flow", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "K" }, { "expr": "sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"bytes_read\"}[1m])) + sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"iter_bytes_read\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "read-flow", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "RaftDB frontend flow has jitter ? (read-flow may hit page-cache)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "Bps", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "compaction": "green", "flow": "rgb(198, 25, 153)", "grpc-kv_commit": "green", "kv_commit": "green", "read-flow": "light-red", "total": "orange", "total-flow": "semi-dark-red", "write-flow": "rgb(0, 202, 255)", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 89 }, "id": 9101, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "read-flow", "stack": true, "transform": "negative-Y", "yaxis": 2, "zindex": 3 }, { "alias": "write-flow", "stack": true, "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"wal_file_bytes\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-flow", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "K" }, { "expr": "sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"bytes_read\"}[1m])) + sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"iter_bytes_read\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "read-flow", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB frontend flow has jitter ? (read-flow may hit page-cache)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "Bps", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "compaction": "green", "flow": "light-red", "grpc-kv_commit": "green", "kv_commit": "green", "read-flow": "light-red", "total": "orange", "total-flow": "semi-dark-red", "write-flow": "rgb(0, 202, 255)", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 94 }, "id": 8025, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "write-flow", "stack": true, "transform": "negative-Y", "yaxis": 2 }, { "alias": "read-flow", "stack": true, "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"flush_write_bytes\"}[1m])) + sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"flush_write_bytes\"}[1m])) + sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"bytes_written\"}[1m])) + sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"bytes_written\"}[1m])) + sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"wal_file_bytes\"}[1m])) + sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"wal_file_bytes\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-flow", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "K" }, { "expr": "sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"bytes_read\"}[1m])) + sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"bytes_read\"}[1m])) + sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"bytes_read\"}[1m])) + sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"bytes_read\"}[1m])) + sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"iter_bytes_read\"}[1m])) + sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"iter_bytes_read\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "read-flow", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total IO flow has jitter ? (read-flow may hit page-cache)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "Bps", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "compaction": "green", "flow": "light-red", "grpc-kv_commit": "green", "kv_commit": "green", "read-flow": "light-red", "total": "orange", "total-flow": "semi-dark-red", "write-flow": "rgb(0, 202, 255)", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 94 }, "id": 9100, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "read-flow", "stack": true, "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-flow", "stack": true, "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"flush_write_bytes\"}[1m])) + sum(rate(tikv_engine_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"flush_write_bytes\"}[1m])) + sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"bytes_written\"}[1m])) + sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"bytes_written\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-flow", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "K" }, { "expr": "sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"bytes_read\"}[1m])) + sum(rate(tikv_engine_compaction_flow_bytes{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\", type=\"bytes_read\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "read-flow", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total compaction flow has jitter ? (read-flow may hit page-cache)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "Bps", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "kv_commit": "green", "kv_pessimistic_lock": "rgba(255, 29, 6, 0.84)", "kv_prewrite": "rgb(11, 38, 239)", "neg-raftdb-wal-duration": "rgb(226, 49, 85)", "raftdb-wal-duration": "semi-dark-red", "raftdb-wal-duration-99%": "rgb(1, 255, 180)", "raftdb-wal-duration-max": "light-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 99 }, "id": 9102, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "raftdb-wal-duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "raftdb-wal-duration-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "intervalFactor": 1, "legendFormat": "write-rpc", "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 }, { "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"wal_file_sync_percentile99\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "raftdb-wal-duration-99%", "refId": "B" }, { "expr": "avg(tikv_engine_wal_file_sync_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"raft\",type=\"wal_file_sync_max\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "raftdb-wal-duration-max", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "RaftDB sync WAL too slow ?", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "µs", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 104 }, "id": 8792, "panels": [], "title": "TiKV-Read affected TiDB-Write ?", "type": "row" }, { "aliasColors": { "copr-duration-99%": "dark-orange", "copr-duration-999%": "light-orange", "handle-duration-99%": "dark-orange", "handle-duration-999%": "light-orange", "index-req": "dark-orange", "kv_commit": "green", "kv_pessimistic_lock": "rgba(255, 29, 6, 0.84)", "kv_prewrite": "rgb(11, 38, 239)", "neg-raftdb-wal-duration": "rgb(226, 49, 85)", "raftdb-wal-duration": "semi-dark-red", "read-rpc-duration-99%": "semi-dark-orange", "read-rpc-duration-999%": "dark-yellow", "select-req": "super-light-orange", "write-rpc": "green", "write-rpc-99%": "semi-dark-red", "write-rpc-999%": "semi-dark-red", "write-rpc-duration": "semi-dark-red" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fill": 2, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 105 }, "id": 10942, "legend": { "alignAsTable": false, "avg": false, "current": false, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "current", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [ { "alias": "index-req", "transform": "negative-Y", "yaxis": 2 }, { "alias": "select-req", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "instant": false, "intervalFactor": 1, "legendFormat": "write-rpc", "metric": "tikv_grpc_msg_duration_seconds_bucket", "refId": "A", "step": 10 }, { "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"index\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "index-req", "refId": "B" }, { "expr": "sum(rate(tikv_coprocessor_request_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=\"select\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "select-req", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Too much coprocessor requests ?", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "ops", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "index-dur-99%": "dark-orange", "index-dur-max": "dark-orange", "index-duration-99%": "dark-orange", "index-duration-max": "dark-orange", "kv_commit": "green", "select+index-count": "dark-purple", "select-dur-99%": "super-light-orange", "select-dur-max": "super-light-orange", "select-duration-99%": "super-light-orange", "select-duration-max": "super-light-orange", "wait-queue": "semi-dark-purple", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 105 }, "id": 10334, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "index-dur-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "select-dur-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "select-dur-max", "yaxis": 2 }, { "alias": "index-dur-max", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"index\"}[1m])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "index-dur-99%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"select\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "select-dur-99%", "refId": "B" }, { "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"select\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "select-dur-max", "refId": "C" }, { "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_handle_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"index\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "index-dur-max", "refId": "E" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Coprocessor handle too slow ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "duration-99%": "rgb(127, 3, 185)", "duration-999%": "super-light-orange", "duration-max": "super-light-purple", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 110 }, "id": 9563, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "duration-999%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "duration-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"select|index\"}[1m])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "duration-99%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "histogram_quantile(0.999, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"select|index\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 1, "legendFormat": "duration-999%", "refId": "B" }, { "expr": "histogram_quantile(1, sum(rate(tikv_coprocessor_request_wait_seconds_bucket{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"select|index\"}[1m])) by (le))", "format": "time_series", "hide": true, "intervalFactor": 1, "legendFormat": "duration-max", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Wait for available coprocessor threads ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "s", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "kv_commit": "green", "wait-queue": "semi-dark-purple", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 110 }, "id": 10790, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "wait-queue", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_futurepool_pending_task_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", name=~\"cop-normal\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "wait-queue", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Wait for available coprocessor threads ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "copr-scan-number": "light-red", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 115 }, "id": 9561, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "copr-scan-number", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "(sum(rate(tikv_coprocessor_scan_details{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"select|index\", cf=\"default\"}[1m])) + sum(rate(tikv_coprocessor_scan_details{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", req=~\"select|index\", cf=\"write\"}[1m])))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "copr-scan-number", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Coprocessor scan too much ? (include MVCC)", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "duration-99%": "dark-blue", "duration-999%": "yellow", "duration-max": "super-light-blue", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 115 }, "id": 10030, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "duration-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_get_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"get_percentile99\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "duration-99%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "avg(tikv_engine_get_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"get_max\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "duration-max", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB read too slow ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "µs", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "kv_commit": "green", "kvdb-delete-skipped-count": "light-red", "wait-queue": "light-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 120 }, "id": 10182, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "kvdb-delete-skipped-count", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_coprocessor_rocksdb_perf{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\" ,metric=\"internal_delete_skipped_count\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "kvdb-delete-skipped-count", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB scan too much deleted data ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "duration-99%": "dark-yellow", "duration-999%": "yellow", "duration-max": "light-blue", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 120 }, "id": 9569, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "duration-max", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_sst_read_micros{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"sst_read_micros_percentile99\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "duration-99%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "avg(tikv_engine_sst_read_micros{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"sst_read_micros_max\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "duration-max", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB read SST too slow ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "µs", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "data-miss-rate": "dark-orange", "duration-99%": "blue", "duration-999%": "yellow", "fall-to-raft-rate": "dark-orange", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 125 }, "id": 11259, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "fall-to-raft-rate", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "sum(rate(tikv_raftstore_proposal_total{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"local_read|read_index\"}[1m])) / sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_commit|kv_prewrite|kv_pessimistic_lock|kv_get|kv_scan|kv_batch_get|kv_batch_get_command|coprocessor\"}[1m]))", "format": "time_series", "intervalFactor": 1, "legendFormat": "fall-to-raft-rate", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Too much get-request fall to raftstore ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "percentunit", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "data-miss-rate": "dark-orange", "duration-99%": "blue", "duration-999%": "yellow", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 125 }, "id": 9570, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "data-miss-rate", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "sum(rate(tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"block_cache_data_miss\"}[1m])) / (sum(rate(tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=\"block_cache_data_hit\"}[1m])) + sum(rate(tikv_engine_cache_efficiency{tidb_cluster=\"$tidb_cluster\", db=\"kv\", type=\"block_cache_data_miss\"}[1m])))", "format": "time_series", "intervalFactor": 1, "legendFormat": "data-miss-rate", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB block cache miss ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "percentunit", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "count": "dark-orange", "hit-count": "dark-orange", "kv_commit": "green", "wait-queue": "orange", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 130 }, "id": 9568, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "count", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_locate{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=\"number_db_next\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "count", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB seek too much times ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "duration-99%": "dark-red", "duration-999%": "yellow", "duration-max": "super-light-blue", "kv_commit": "green", "wait-duration-999%": "orange", "wait-queue": "semi-dark-red", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 12, "y": 130 }, "id": 9567, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "duration-99%", "transform": "negative-Y", "yaxis": 2 }, { "alias": "duration-max", "transform": "negative-Y", "yaxis": 2 }, { "alias": "write-rpc", "zindex": -3 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "avg(tikv_engine_seek_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"seek_percentile99\"})", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "duration-99%", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" }, { "expr": "avg(tikv_engine_seek_micro_seconds{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\",type=\"seek_max\"})", "format": "time_series", "intervalFactor": 1, "legendFormat": "duration-max", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB seek too slow ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "µs", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "hit-count": "super-light-blue", "kv_commit": "green", "wait-queue": "orange", "wait-sched": "semi-dark-red", "write-rpc": "green" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "editable": true, "error": false, "fill": 2, "grid": {}, "gridPos": { "h": 5, "w": 12, "x": 0, "y": 135 }, "id": 9566, "legend": { "alignAsTable": false, "avg": false, "current": false, "hideEmpty": true, "max": false, "min": false, "rightSide": false, "show": true, "sideWidth": null, "sort": "max", "sortDesc": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "paceLength": 10, "percentage": false, "pointradius": 5, "points": false, "renderer": "flot", "repeatedByRow": true, "seriesOverrides": [ { "alias": "hit-count", "transform": "negative-Y", "yaxis": 2 } ], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum(rate(tikv_engine_get_served{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", db=\"kv\", type=~\"get_hit_l2_and_up|get_hit_l1\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "hit-count", "refId": "D" }, { "expr": "sum(rate(tikv_grpc_msg_duration_seconds_count{tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", type=~\"kv_prewrite|kv_commit|kv_pessimistic_lock\"}[1m]))", "format": "time_series", "hide": false, "intervalFactor": 1, "legendFormat": "write-rpc", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "KVDB read too much SSTs ?", "tooltip": { "msResolution": false, "shared": true, "sort": 0, "value_type": "cumulative" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": "0", "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "refresh": "5m", "schemaVersion": 18, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": { }, "datasource": "${DS_TEST-CLUSTER}", "hide": 2, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", "options": [ ], "query": "label_values(tikv_engine_block_cache_size_bytes, tidb_cluster)", "refresh": 2, "regex": "", "sort": 1, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": ".*", "current": {}, "datasource": "${DS_TEST-CLUSTER}", "definition": "", "hide": 0, "includeAll": true, "label": "Instance", "multi": true, "name": "instance", "options": [], "query": "label_values(tikv_engine_size_bytes{tidb_cluster=\"$tidb_cluster\"}, instance)", "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "browser", "title": "Test-Cluster-TiKV-FastTune", "uid": "TiKVFastTune", "version": 7 }