{ "__inputs": [ { "name": "DS_TEST-CLUSTER", "label": "${DS_TEST-CLUSTER}", "description": "", "type": "datasource", "pluginId": "prometheus", "pluginName": "Prometheus" } ], "__requires": [ { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "7.5.11" }, { "type": "panel", "id": "graph", "name": "Graph", "version": "" }, { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" } ], "annotations": { "list": [ { "builtIn": 1, "datasource": "${DS_TEST-CLUSTER}", "enable": true, "hide": false, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "editable": true, "gnetId": null, "graphTooltip": 0, "id": null, "iteration": 1653965508897, "links": [], "panels": [ { "cacheTimeout": null, "datasource": "${DS_TEST-CLUSTER}", "description": "", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 8, "gradientMode": "none", "hideFrom": { "graph": false, "legend": false, "tooltip": false }, "lineInterpolation": "stepBefore", "lineStyle": { "fill": "solid" }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false }, "decimals": 0, "mappings": [ { "from": "", "id": 1, "text": "Invaild", "to": "", "type": 1, "value": "0" }, { "from": "", "id": 2, "text": "New", "to": "", "type": 1, "value": "1" }, { "from": "", "id": 3, "text": "Running", "to": "", "type": 1, "value": "2" }, { "from": "", "id": 4, "text": "Paused", "to": "", "type": 1, "value": "3" }, { "from": "", "id": 5, "text": "Stopped", "to": "", "type": 1, "value": "4" }, { "from": "", "id": 6, "text": "Finished", "to": "", "type": 1, "value": "5" }, { "from": "", "id": 7, "text": "Error", "to": "", "type": 1, "value": "15" } ], "min": 0, "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "#EAB839", "value": 0 }, { "color": "light-green", "value": 1 }, { "color": "super-light-green", "value": 2 }, { "color": "light-red", "value": 3 }, { "color": "semi-dark-red", "value": 4 }, { "color": "dark-red", "value": 5 } ] }, "unit": "short" }, "overrides": [] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 0 }, "id": 104, "links": [], "options": { "graph": {}, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltipOptions": { "mode": "single" } }, "pluginVersion": "7.5.7", "targets": [ { "exemplar": true, "expr": "dm_worker_task_state{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 1, "legendFormat": "{{task}}-{{source_id}}-{{worker}}", "refId": "A" } ], "timeFrom": null, "timeShift": null, "title": "Task State", "type": "timeseries" }, { "collapsed": true, "datasource": "${DS_TEST-CLUSTER}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 7 }, "id": 56, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The data import process percentage of dump unit. The value is an estimated value and may exceed less than 100%.", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 8 }, "hiddenSeries": false, "id": 88, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "dumpling_dump_finished_rows{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"} / dumpling_dump_estimate_total_rows{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "dump progress", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The data import process percentage of load unit. The value range is 0% ~ 100%", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 8 }, "hiddenSeries": false, "id": 87, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "lightning_chunks{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",state=\"finished\"} / ignoring(state) lightning_chunks{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",state=\"estimated\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "load progress", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The checksum process percentage of load unit. The value range is 0% ~ 100%", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 8 }, "hiddenSeries": false, "id": 95, "interval": null, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": 100, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "lightning_tables{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",state=\"completed\"} / ignoring(state) lightning_tables{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",state=\"pending\"}", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "checksum progress", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "percentunit", "label": null, "logBase": 1, "max": "1", "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Dumper encounters an error within the DM-worker and exits", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 15 }, "hiddenSeries": false, "id": 32, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "changes(dm_dumpling_exit_with_error_count{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[30m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "dump process exits with error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "load unit encounters an error within the DM-worker and exits", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 15 }, "hiddenSeries": false, "id": 33, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "changes(dm_loader_exit_with_error_count{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[30m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "load process exits with error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "idle load unit workers", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 22 }, "hiddenSeries": false, "id": 18, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "lightning_idle_workers{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{name}} - {{task}} - {{source_id}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Idle workers", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration that load unit process a chunk (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 22 }, "hiddenSeries": false, "id": 96, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.7", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "1/rate(lightning_chunks{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",state=\"finished\"}[1m])", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Chunk process duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 2, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of external resources used by load unit", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 89, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "lightning_kv_encoder{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",type=\"open\"} - ignoring(type) lightning_kv_encoder{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",type=\"closed\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "KV Encoder", "refId": "A" }, { "expr": "lightning_importer_engine{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",type=\"open\"} - ignoring(type) lightning_importer_engine{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",type=\"closed\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "Importer Engines (via Lightning)", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "External resources", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "none", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The statistic of bytes in different status of data files processed by load unit", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 15 }, "hiddenSeries": false, "id": 17, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "lightning_row_read_bytes_sum{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "parser read size - {{task}} - {{source_id}}", "refId": "A" }, { "expr": "lightning_block_deliver_bytes_sum{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{kind}} deliver size - {{task}} - {{source_id}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Total bytes for load unit", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The speed of load unit processing source files internally", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 29 }, "hiddenSeries": false, "id": 16, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(lightning_block_deliver_bytes_sum{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[30s])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{kind}} deliver rate {{task}} - {{source_id}}", "refId": "B" }, { "exemplar": true, "expr": "sum(rate(lightning_block_deliver_bytes_sum{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[30s]))", "format": "time_series", "intervalFactor": 2, "legendFormat": "total deliver rate {{task}} - {{source_id}}", "refId": "A" } ], "title": "SQL process rate", "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "Bps", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ] } ], "repeat": null, "title": "dump and load ", "type": "row" }, { "collapsed": true, "datasource": "${DS_TEST-CLUSTER}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 8 }, "id": 57, "panels": [ { "aliasColors": { "admin queue-flush": "blue" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "number of sql executed per second(one minute average value, dm_syncer_finished_jobs_total )", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 12, "w": 15, "x": 0, "y": 10 }, "hiddenSeries": false, "id": 102, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(dm_syncer_finished_jobs_total{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval]))by (task,source_id,target_schema,type)", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{target_schema}} - {{type}} ", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Replication QPS", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:805", "decimals": 0, "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:806", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The predicted remaining time it takes Syncer to be completely synchronized with the master (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 9, "x": 15, "y": 10 }, "hiddenSeries": false, "id": 37, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_syncer_remaining_time{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Remaining time to sync", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The latency time it takes to replicate the binlog from master to Syncer (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 6, "w": 9, "x": 15, "y": 16 }, "hiddenSeries": false, "id": 38, "interval": null, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "maxDataPoints": 100, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_replication_lag_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,source_id,task))", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - 90", "refId": "A" }, { "exemplar": true, "expr": "dm_syncer_replication_lag_gauge{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - gauge", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Replication lag", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:669", "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:670", "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "admin queue-flush": "blue" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "number of sql executed per second(one minute average value, dm_syncer_finished_jobs_total )\n\nThe maximum TPS estimated according to the recent SQL time-consuming, may vary greatly with different SQL", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 15, "x": 0, "y": 22 }, "hiddenSeries": false, "id": 110, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(dm_syncer_finished_jobs_total{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\"}[$interval])) by (task)", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} ", "refId": "A" }, { "exemplar": true, "expr": "sum(dm_syncer_ideal_qps{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\"}) by (task)", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - ideal", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Replication QPS (Sum by task)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:805", "decimals": 0, "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "$$hashKey": "object:806", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The binlog replication unit process encounters an error within the DM-worker and exits", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 9, "x": 15, "y": 22 }, "hiddenSeries": false, "id": 39, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "changes(dm_syncer_exit_with_error_count{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[30m])", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "process exits with error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": { "admin queue-flush": "blue" }, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of add/finished jobs per unit of time", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 15, "x": 0, "y": 30 }, "hiddenSeries": false, "id": 2, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(rate(dm_syncer_finished_jobs_total{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval])) by (task,source_id,target_schema,type)", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "finished-{{task}} - {{target_schema}} - {{type}} ", "refId": "A" }, { "exemplar": true, "expr": "sum(-rate(dm_syncer_added_jobs_total{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",target_schema=~\"$target_schema\",type=~\"$type\"}[$interval])) by (task,source_id,target_schema,type)", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "add-{{task}} - {{target_schema}} - {{type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "DM SQL jobs ", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:731", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:732", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time it takes binlog replication unit to detect conflicts between DMLs (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 9, "x": 15, "y": 30 }, "hiddenSeries": false, "id": 64, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_conflict_detect_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\"}[$interval])) by (le,task))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - 90", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_conflict_detect_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\"}[$interval])) by (le,task))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - 95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_conflict_detect_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\"}[$interval])) by (le,task))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - 99", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "DML conflict detect duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 2, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The number of binlog events received per unit of time (this number does not include the events that need to be skipped)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 15, "x": 0, "y": 38 }, "hiddenSeries": false, "id": 4, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum by (instance,task,stage) (rate(dm_syncer_binlog_transform_cost_count{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\",source_id=~\"$source\",stage=~\"$stage\"}[$interval]))", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{instance}} - {{stage}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "binlog event QPS", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:325", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:326", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time it takes binlog replication unit to add a job to the queue (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 9, "x": 15, "y": 38 }, "hiddenSeries": false, "id": 63, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum (rate(dm_syncer_add_job_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - 90", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_add_job_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - 95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_add_job_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",queueNo=~\"$queueNo\",type=~\"$type\"}[$interval])) by (task,le))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - 99", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "add job duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time it takes binlog replication unit to execute the transaction to the downstream (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 15, "x": 0, "y": 46 }, "hiddenSeries": false, "id": 60, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_txn_duration_time_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{instance}} - txn-90", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_txn_duration_time_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{instance}} - txn-95", "refId": "B" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_txn_duration_time_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{instance}} - txn-99", "refId": "C" }, { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_query_duration_time_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{instance}} - ddl-90", "refId": "D" }, { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_query_duration_time_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{instance}} - ddl-95", "refId": "E" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_query_duration_time_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{instance}} - ddl-99", "refId": "F" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "transaction/ddl query execution latency", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:401", "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "$$hashKey": "object:402", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The time it takes binlog replication unit to execute every statement to the downstream (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 9, "x": 15, "y": 46 }, "hiddenSeries": false, "id": 1, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",type=\"begin\",instance=~\"$instance\"}[$interval])) by (le,task,instance))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{instance}} - begin", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",type=\"stmt\",instance=~\"$instance\"}[$interval])) by (le,task,instance))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{instance}} - stmt", "refId": "B" }, { "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",type=\"commit\",instance=~\"$instance\"}[$interval])) by (le,task,instance))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{instance}} - commit", "refId": "C" }, { "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_stmt_duration_time_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",type=\"rollback\",instance=~\"$instance\"}[1m])) by (le,task,instance))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{instance}} - rollback", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "statement execution latency - 90", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The number of binlog events received per unit of time that needs to be skipped", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 9, "x": 15, "y": 53 }, "hiddenSeries": false, "id": 35, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "rate(dm_syncer_skip_binlog_duration_count{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",type=\"rows\"}[$interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - rows", "refId": "A" }, { "expr": "rate(dm_syncer_skip_binlog_duration_count{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\",type=\"query\"}[$interval])", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - query", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "skipped binlog event QPS", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 0, "description": "The remain length of DML job queues", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 15, "x": 0, "y": 54 }, "hiddenSeries": false, "id": 62, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(dm_syncer_queue_size{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\"}) by (task)", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "DML queue remain length", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:246", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:247", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The size of a single binlog event that the binlog replication unit reads from relay log or upstream master", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 61 }, "hiddenSeries": false, "id": 61, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "sort": null, "sortDesc": null, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_event_size_bucket{tidb_cluster_id=~\"$tidb_cluster\",worker=~\"$worker\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,worker,task))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{worker}} - 90", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_event_size_bucket{tidb_cluster_id=~\"$tidb_cluster\",worker=~\"$worker\",task=~\"$task\",source_id=~\"$source\"}[1m])) by (le,worker,task))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{worker}} - 95", "refId": "B" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_event_size_bucket{tidb_cluster_id=~\"$tidb_cluster\",worker=~\"$worker\",task=~\"$task\",source_id=~\"$source\"}[1m])) by (le,worker,task))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{worker}} - 99", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "binlog event size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1109", "decimals": 1, "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1110", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of binlog files in binlog replication unit that are behind the master", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 7, "x": 8, "y": 61 }, "hiddenSeries": false, "id": 43, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "clamp_min(dm_syncer_binlog_file{tidb_cluster_id=~\"$tidb_cluster\",source_id=~\"$source\",task=~\"$task\",node=\"master\"} - ON(task,source_id) dm_syncer_binlog_file{tidb_cluster_id=~\"$tidb_cluster\",source_id=~\"$source\",task=~\"$task\",node=\"syncer\"}, 0)", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "binlog file gap between master and syncer", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The number of rows of each binlog event that enters the DM", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 9, "x": 15, "y": 61 }, "hiddenSeries": false, "id": 106, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_event_row_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source))", "hide": false, "interval": "", "legendFormat": "{{task}}-{{source}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "binlog event row number", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The duration that the binlog replication unit reads binlog event from the relay log or upstream MySQL (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 69 }, "hiddenSeries": false, "id": 36, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_read_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - 90", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_read_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - 95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_read_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - 99", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "read binlog event duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The time it takes binlog replication unit to parse and transform the binlog into SQL statements (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 7, "x": 8, "y": 69 }, "hiddenSeries": false, "id": 58, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_transform_cost_bucket{tidb_cluster_id=~\"$tidb_cluster\",worker=~\"$worker\",task=~\"$task\",source_id=~\"$source\"}[1m])) by (le,worker,task,stage))", "format": "time_series", "hide": false, "interval": "", "intervalFactor": 2, "legendFormat": "90-[[worker]]-[[task]]-{{stage}}", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_transform_cost_bucket{tidb_cluster_id=~\"$tidb_cluster\",worker=~\"$worker\",task=~\"$task\",source_id=~\"$source\"}[1m])) by (le,worker,task,stage))", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 2, "legendFormat": "95-[[worker]]-[[task]]-{{stage}}", "refId": "B" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_transform_cost_bucket{tidb_cluster_id=~\"$tidb_cluster\",worker=~\"$worker\",task=~\"$task\",source_id=~\"$source\"}[1m])) by (le,worker,task,stage))", "format": "time_series", "hide": true, "interval": "", "intervalFactor": 2, "legendFormat": "99-[[worker]]-[[task]]-{{stage}}", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "binlog event cost duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 2, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The time it takes binlog replication unit to dispatch a binlog event (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 9, "x": 15, "y": 69 }, "hiddenSeries": false, "id": 59, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - 90", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - 95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_dispatch_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - 99", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "dispatch binlog event duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The number of unsynced tables in the subtask", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 77 }, "hiddenSeries": false, "id": 45, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_syncer_unsynced_table_number{tidb_cluster_id=~\"$tidb_cluster\",source_id=~\"$source\",task=~\"$task\"}", "format": "time_series", "instant": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "unsynced tables", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The number of binlog files in binlog replication unit that are behind relay", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 7, "x": 8, "y": 77 }, "hiddenSeries": false, "id": 44, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_relay_binlog_file{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",node=\"relay\"} - ON(instance) group_right dm_syncer_binlog_file{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\",node=\"syncer\"}", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{instance}} - {{task}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "binlog file gap between relay and syncer", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time interval of DM written to disk", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 9, "x": 15, "y": 77 }, "hiddenSeries": false, "id": 107, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_flush_checkpoints_time_interval_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\"}[$interval])) by (le,task,worker))", "hide": false, "interval": "", "legendFormat": "{{task}}-{{worker}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Flushed Checkpoint Time Interval", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:644", "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "$$hashKey": "object:645", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "Is waiting shard DDL lock to be resolved, >0 means waiting", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 8, "x": 0, "y": 85 }, "hiddenSeries": false, "id": 46, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_syncer_shard_lock_resolving{tidb_cluster_id=~\"$tidb_cluster\",source_id=~\"$source\",task=~\"$task\"}", "format": "time_series", "instant": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "shard lock resolving", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": "", "logBase": 1, "max": "1", "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The time it takes binlog replication unit to skip a binlog event (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 7, "x": 8, "y": 85 }, "hiddenSeries": false, "id": 65, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_skip_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - 90", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_skip_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - 95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_skip_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\",source_id=~\"$source\"}[$interval])) by (le,task,source_id))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - 99", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "skip event duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "Display the number of sql contained in the dm copy incremental data to the downstream transaction from each latitude", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 9, "x": 15, "y": 85 }, "hiddenSeries": false, "id": 105, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_replication_transaction_batch_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task,queueNo,type))", "hide": false, "interval": "", "legendFormat": "{{task}} - {{queueNo}} - {{type}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "replication transaction batch", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:581", "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "$$hashKey": "object:582", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 15, "x": 0, "y": 93 }, "hiddenSeries": false, "id": 109, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": true, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.90, sum(rate(dm_syncer_binlog_transform_cost_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - {{instance}} - 90", "refId": "A" }, { "exemplar": true, "expr": "histogram_quantile(0.95, sum(rate(dm_syncer_binlog_transform_cost_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - {{instance}} - 95", "refId": "B" }, { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(dm_syncer_binlog_transform_cost_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",task=~\"$task\"}[$interval])) by (le,instance,task))", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{task}} - {{source_id}} - {{instance}} - 99", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": " transform binlog event duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:659", "decimals": 1, "format": "short", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "$$hashKey": "object:660", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "binlog replication", "type": "row" }, { "collapsed": true, "datasource": "${DS_TEST-CLUSTER}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 9 }, "id": 55, "panels": [ { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The storage capacity of the disk occupied by the relay log", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 11 }, "hiddenSeries": false, "id": 20, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_relay_space{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",type=\"capacity\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "storage capacity", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 1, "description": "The remaining storage capacity of the disk occupied by the relay log", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 11 }, "hiddenSeries": false, "id": 21, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_relay_space{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",type=\"available\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "storage remain", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 1, "format": "bytes", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": 0, "description": "The relay log encounters an error within the DM-worker and exits", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 11 }, "hiddenSeries": false, "id": 30, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null as zero", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_relay_exit_with_error_count{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": " {{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "process exits with error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": "0", "show": true }, { "decimals": null, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "decimals": null, "description": "The number of corrupted relay log files", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 11 }, "hiddenSeries": false, "id": 31, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_relay_data_corruption{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "relay log data corruption", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of errors encountered when the relay log reads the binlog from the upstream MySQL", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 18 }, "hiddenSeries": false, "id": 28, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_relay_read_error_count{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "fail to read binlog from master", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": null, "format": "short", "label": "", "logBase": 1, "max": null, "min": "0", "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "cacheTimeout": null, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of errors encountered when the relay log writes the binlog to disks", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 18 }, "hiddenSeries": false, "id": 29, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_relay_write_error_count{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "fail to write relay log", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The largest index number of relay log files. E.g. \"value = 1\" indicates \"mysql-bin.000001\"", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 18 }, "hiddenSeries": false, "id": 7, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_relay_binlog_file{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",node=\"master\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "master", "refId": "A" }, { "expr": "dm_relay_binlog_file{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",node=\"relay\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "relay", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "binlog file index", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of binlog files in the relay log that are behind the upstream master", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 18 }, "hiddenSeries": false, "id": 42, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_relay_binlog_file{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",node=\"master\"} - ON(instance) dm_relay_binlog_file{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",node=\"relay\"}", "format": "time_series", "hide": false, "interval": "$interval", "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "binlog file gap between master and relay", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The write offset of the latest relay log file", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 25 }, "hiddenSeries": false, "id": 6, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": false, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_relay_binlog_pos{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",node=\"master\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "master - {{instance}}", "refId": "A" }, { "expr": "dm_relay_binlog_pos{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\",node=\"relay\"}", "format": "time_series", "interval": "$interval", "intervalFactor": 2, "legendFormat": "relay - {{instance}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "binlog pos", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration that the relay log reads binlog event from the upstream MySQL (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 25 }, "hiddenSeries": false, "id": 9, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(dm_relay_read_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "interval": "", "intervalFactor": 2, "legendFormat": "90", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(dm_relay_read_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(dm_relay_read_binlog_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "read binlog event duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 2, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The duration that the relay log writes binlog event into the disks each time (in seconds)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 25 }, "hiddenSeries": false, "id": 12, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(dm_relay_write_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(dm_relay_write_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(dm_relay_write_duration_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "write relay log duration", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 2, "format": "s", "label": null, "logBase": 1, "max": null, "min": 0, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The size of a single binlog event that the relay log writes into the disks", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 25 }, "hiddenSeries": false, "id": 25, "legend": { "alignAsTable": true, "avg": false, "current": false, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(dm_relay_write_size_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "90", "refId": "A" }, { "expr": "histogram_quantile(0.95, sum(rate(dm_relay_write_size_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "95", "refId": "B" }, { "expr": "histogram_quantile(0.99, sum(rate(dm_relay_write_size_bucket{tidb_cluster_id=~\"$tidb_cluster\",instance=~\"$instance\"}[1m])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "99", "refId": "C" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "binlog event size", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 2, "format": "decbytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "relay log", "type": "row" }, { "collapsed": true, "datasource": "${DS_TEST-CLUSTER}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 10 }, "id": 80, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "number of dm-masters try to start leader components per minute", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 86, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_master_start_leader_counter{tidb_cluster_id=~\"$tidb_cluster\"}", "format": "time_series", "instant": false, "intervalFactor": 2, "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "number of dm-masters start leader components per minute", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "number of worker in different states, -1 - unrecognized, 0 - offline, 1 - free, 2 - bound", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 12 }, "hiddenSeries": false, "id": 83, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "count_values(\"state\", dm_master_worker_state{tidb_cluster_id=~\"$tidb_cluster\"})", "format": "time_series", "instant": false, "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "number of workers in different state", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "state of worker, -1 - unrecognized, 0 - offline, 1 - free, 2 - bound", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 12 }, "hiddenSeries": false, "id": 81, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (worker) (dm_master_worker_state{tidb_cluster_id=~\"$tidb_cluster\"})", "format": "time_series", "instant": false, "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "workers' state", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "number of worker event error", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 12 }, "hiddenSeries": false, "id": 85, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (type) (dm_master_worker_event_error{tidb_cluster_id=~\"$tidb_cluster\"})", "format": "time_series", "instant": false, "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "number of worker event error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "number of shard DDL lock/operation error in one minute", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 82, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (task, type) (increase(dm_master_shard_ddl_error{tidb_cluster_id=~\"$tidb_cluster\"}[1m]))", "format": "time_series", "instant": false, "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "shard ddl error per minute", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "number of pending shard DDL in different states, Un-synced (waiting all upstream), Synced (all upstream finished, waiting all downstream)", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 19 }, "hiddenSeries": false, "id": 84, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "sum by (task, type) (dm_master_ddl_state_number{tidb_cluster_id=~\"$tidb_cluster\"})", "format": "time_series", "instant": false, "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "number of pending shard ddl", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "repeat": null, "title": "HA", "type": "row" }, { "collapsed": true, "datasource": "${DS_TEST-CLUSTER}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 11 }, "id": 67, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of error happens before operate", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 28 }, "hiddenSeries": false, "id": 71, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_worker_operate_error{tidb_cluster_id=~\"$tidb_cluster\",type=\"BeforeAnyOp\"}", "format": "time_series", "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "before any operate error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of error happens in operate source bound", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 28 }, "hiddenSeries": false, "id": 72, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_worker_operate_error{tidb_cluster_id=~\"$tidb_cluster\",type=\"SourceBound\"}", "format": "time_series", "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "source bound error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of error happens in operate start", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 28 }, "hiddenSeries": false, "id": 76, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_worker_operate_error{tidb_cluster_id=~\"$tidb_cluster\",type=\"Start\"}", "format": "time_series", "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "start error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of error happens in operate pause", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 28 }, "hiddenSeries": false, "id": 74, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_worker_operate_error{tidb_cluster_id=~\"$tidb_cluster\",type=\"Pause\"}", "format": "time_series", "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "pause error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of error happens in operate resume", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 0, "y": 35 }, "hiddenSeries": false, "id": 78, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_worker_operate_error{tidb_cluster_id=~\"$tidb_cluster\",type=\"Resume\"}", "format": "time_series", "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "resume error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of error happens in operate auto-resume", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 6, "y": 35 }, "hiddenSeries": false, "id": 69, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_worker_operate_error{tidb_cluster_id=~\"$tidb_cluster\",type=\"AutoResume\"}", "format": "time_series", "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "auto-resume error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of error happens in operate update", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 12, "y": 35 }, "hiddenSeries": false, "id": 77, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_worker_operate_error{tidb_cluster_id=~\"$tidb_cluster\",type=\"Update\"}", "format": "time_series", "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "update error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "description": "The number of error happens in operate stop", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 7, "w": 6, "x": 18, "y": 35 }, "hiddenSeries": false, "id": 75, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 5, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "expr": "dm_worker_operate_error{tidb_cluster_id=~\"$tidb_cluster\",type=\"Stop\"}", "format": "time_series", "intervalFactor": 2, "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "stop error", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "decimals": 0, "format": "short", "label": null, "logBase": 1, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": false } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "operate error", "type": "row" }, { "collapsed": true, "datasource": "${DS_TEST-CLUSTER}", "gridPos": { "h": 1, "w": 24, "x": 0, "y": 12 }, "id": 122, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 13 }, "hiddenSeries": false, "id": 120, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "dm_validator_validator_error_count{tidb_cluster_id=~\"$tidb_cluster\"}", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Validator Error Count", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 21 }, "hiddenSeries": false, "id": 116, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "dm_validator_validator_logpos_latency{tidb_cluster_id=~\"$tidb_cluster\"}", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Validator Binlog Latency (in byte)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 21 }, "hiddenSeries": false, "id": 114, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "dm_validator_validator_binlog_file{tidb_cluster_id=~\"$tidb_cluster\"}", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Validator Binlog Position (in file)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 29 }, "hiddenSeries": false, "id": 118, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "dm_validator_validator_logfile_latency{tidb_cluster_id=~\"$tidb_cluster\"}", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Validator Binlog Latency (in file)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": "${DS_TEST-CLUSTER}", "fieldConfig": { "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 29 }, "hiddenSeries": false, "id": 112, "legend": { "avg": false, "current": false, "max": false, "min": false, "show": false, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "7.5.11", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "dm_validator_validator_binlog_pos{tidb_cluster_id=~\"$tidb_cluster\"}", "interval": "", "legendFormat": "", "queryType": "randomWalk", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Validator Binlog Position (in byte)", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Validator Statistics", "type": "row" } ], "refresh": false, "schemaVersion": 27, "style": "dark", "tags": [], "templating": { "list": [ { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(go_goroutines, tidb_cluster_id)", "description": null, "error": null, "hide": 0, "includeAll": false, "label": "tidb_cluster", "multi": false, "name": "tidb_cluster", "options": [], "query": { "query": "label_values(go_goroutines, tidb_cluster_id)", "refId": "debug-tidb_cluster-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(dm_worker_task_state, task)", "description": null, "error": null, "hide": 0, "includeAll": true, "label": "Task", "multi": true, "name": "task", "options": [], "query": { "query": "label_values(dm_worker_task_state, task)", "refId": "prometheus-local-task-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(dm_worker_task_state, source_id)", "description": null, "error": null, "hide": 0, "includeAll": true, "label": "Source", "multi": true, "name": "source", "options": [], "query": { "query": "label_values(dm_worker_task_state, source_id)", "refId": "prometheus-local-source-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 1, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(dm_worker_task_state{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\"}, instance)", "description": null, "error": null, "hide": 0, "includeAll": true, "label": "Instance", "multi": true, "name": "instance", "options": [], "query": { "query": "label_values(dm_worker_task_state{tidb_cluster_id=~\"$tidb_cluster\",task=~\"$task\"}, instance)", "refId": "prometheus-local-instance-Variable-Query" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(dm_syncer_finished_jobs_total, queueNo)", "description": null, "error": null, "hide": 2, "includeAll": true, "label": "Queue", "multi": true, "name": "queueNo", "options": [], "query": { "query": "label_values(dm_syncer_finished_jobs_total, queueNo)", "refId": "prometheus-local-queueNo-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(dm_syncer_finished_jobs_total,type)", "description": null, "error": null, "hide": 0, "includeAll": true, "label": "Type", "multi": true, "name": "type", "options": [], "query": { "query": "label_values(dm_syncer_finished_jobs_total,type)", "refId": "prometheus-local-type-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(dm_syncer_binlog_transform_cost_count,stage)", "description": null, "error": null, "hide": 2, "includeAll": true, "label": "Stage", "multi": true, "name": "stage", "options": [], "query": { "query": "label_values(dm_syncer_binlog_transform_cost_count,stage)", "refId": "prometheus-local-stage-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(dm_syncer_finished_jobs_total, target_schema)", "description": null, "error": null, "hide": 0, "includeAll": true, "label": "Target Schema", "multi": true, "name": "target_schema", "options": [], "query": { "query": "label_values(dm_syncer_finished_jobs_total, target_schema)", "refId": "prometheus-local-target_schema-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(dm_syncer_finished_jobs_total, target_table)", "description": null, "error": null, "hide": 2, "includeAll": true, "label": "Target_table", "multi": true, "name": "target_table", "options": [], "query": { "query": "label_values(dm_syncer_finished_jobs_total, target_table)", "refId": "prometheus-local-target_table-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { "auto": true, "auto_count": 30, "auto_min": "10s", "current": { "selected": false, "text": "auto", "value": "$__auto_interval_interval" }, "description": null, "error": null, "hide": 0, "label": "Interval", "name": "interval", "options": [ { "selected": true, "text": "auto", "value": "$__auto_interval_interval" }, { "selected": false, "text": "30s", "value": "30s" }, { "selected": false, "text": "1m", "value": "1m" }, { "selected": false, "text": "10m", "value": "10m" }, { "selected": false, "text": "30m", "value": "30m" }, { "selected": false, "text": "1h", "value": "1h" }, { "selected": false, "text": "6h", "value": "6h" }, { "selected": false, "text": "12h", "value": "12h" }, { "selected": false, "text": "1d", "value": "1d" }, { "selected": false, "text": "7d", "value": "7d" }, { "selected": false, "text": "14d", "value": "14d" }, { "selected": false, "text": "30d", "value": "30d" } ], "query": "30s,1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", "refresh": 2, "skipUrlSync": false, "type": "interval" }, { "allValue": null, "current": {}, "datasource": "${DS_TEST-CLUSTER}", "definition": "label_values(dm_syncer_finished_jobs_total,worker)", "description": null, "error": null, "hide": 2, "includeAll": true, "label": "Worker", "multi": true, "name": "worker", "options": [], "query": { "query": "label_values(dm_syncer_finished_jobs_total,worker)", "refId": "prometheus-local-worker-Variable-Query" }, "refresh": 2, "regex": "", "skipUrlSync": false, "sort": 0, "tagValuesQuery": "", "tags": [], "tagsQuery": "", "type": "query", "useTags": false } ] }, "time": { "from": "now-1h", "to": "now" }, "timepicker": { "refresh_intervals": [ "5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d" ], "time_options": [ "5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d" ] }, "timezone": "", "title": "[SharedScope]DM-Monitor-Professional", "uid": "Ja68YqRnz", "version": 13 }