{ "annotations": { "list": [ { "builtIn": 1, "datasource": "-- Grafana --", "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", "type": "dashboard" } ] }, "description": "Troubleshoot Dashboard", "editable": true, "gnetId": null, "graphTooltip": 0, "iteration": 1637581953227, "links": [ { "asDropdown": false, "icon": "dashboard", "includeVars": true, "keepTime": false, "tags": [], "targetBlank": true, "title": "TiDB In One screen", "tooltip": "", "type": "link", "url": "./d/free-tier-one-screen-tidb/free-tier-tidb-in-one-screen?orgId=1&refresh=1m&var-tenant_name=${tenant_name}&var-tenant_id=${tenant_id}&var-cluster_id=${cluster_id}" }, { "asDropdown": false, "icon": "dashboard", "includeVars": true, "keepTime": false, "tags": [], "targetBlank": true, "title": "Kubernetes Compute Resource Usage", "tooltip": "", "type": "link", "url": "./d/efa86fd1d0c121a26444b636a3f509a8/kubernetes-compute-resources-cluster?orgId=1&refresh=10s&var-cluster=freetier/eks/${cluster_k8s}&var-datasource=default" } ], "panels": [ { "collapsed": false, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, "id": 37, "panels": [], "title": "Abnormal TiDB Cluster Pods", "type": "row" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 1 }, "hiddenSeries": false, "id": 35, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(increase(kube_pod_container_status_restarts_total{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\".*$namespace\"}[$__range])) by (pod)", "hide": false, "interval": "", "legendFormat": "", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pod Restarted Over Selected Range", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1797", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1798", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 1 }, "hiddenSeries": false, "id": 39, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(kube_pod_container_status_last_terminated_reason{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\".*$namespace\", reason!=\"Completed\"}) by (pod,reason)", "hide": false, "interval": "", "legendFormat": "{{pod}}: {{reason}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pod Last Terminated Reason", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1968", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1969", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 9 }, "hiddenSeries": false, "id": 41, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(kube_pod_status_phase{cluster=\"freetier/eks/$cluster_k8s\",namespace=~\".*$namespace\",phase!~\"Running|Succeeded\"}) by (pod, phase)", "hide": false, "interval": "", "legendFormat": "{{pod}}: {{phase}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pod Abnormal Phase", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1525", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1526", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 9 }, "hiddenSeries": false, "id": 43, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{namespace=~\".*$namespace\",cluster=\"freetier/eks/$cluster_k8s\"}[5m])) by (pod, namespace)/\nsum(increase(container_cpu_cfs_periods_total{namespace=~\".*$namespace\",cluster=\"freetier/eks/$cluster_k8s\"}[5m])) by (pod, namespace)", "hide": false, "interval": "", "legendFormat": "{{pod}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Throttle", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:858", "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:859", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 17 }, "id": 31, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "The pod will be shown in this panel only if pods have restarted over selected range time.", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 2 }, "hiddenSeries": false, "id": 27, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideEmpty": false, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 350, "sort": "current", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(increase(kube_pod_container_status_restarts_total{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}[$__range])) by (namespace,pod)", "interval": "", "legendFormat": "{{namespace}}/{{pod}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pod Restarted Over Selected Range", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1910", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1911", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 2 }, "hiddenSeries": false, "id": 29, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(kube_pod_container_status_last_terminated_reason{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\", reason!=\"Completed\"}) by (namespace,pod,reason)", "hide": true, "interval": "", "legendFormat": "{{pod}}: {{reason}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pod Last Terminated Reason", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2764", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2765", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, "hiddenSeries": false, "id": 33, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(kube_pod_status_phase{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\",phase!=\"Running\"}) by (namespace,pod, phase)", "hide": false, "interval": "", "legendFormat": "{{namespace}}/{{pod}}: {{phase}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pod Abnormal Phase", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:1525", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:1526", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, "hiddenSeries": false, "id": 25, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "sideWidth": 400, "sort": "max", "sortDesc": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(increase(container_cpu_cfs_throttled_periods_total{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}[5m])) by (pod, namespace)/\nsum(increase(container_cpu_cfs_periods_total{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}[5m])) by (pod, namespace)", "hide": false, "interval": "", "legendFormat": "{{namespace}}/{{pod}}", "refId": "A" } ], "thresholds": [ { "$$hashKey": "object:3081", "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.5, "yaxis": "left" } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Throttle", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:858", "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:859", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Abnormal System Pods", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 18 }, "id": 21, "panels": [ { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "filterable": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "status" }, "properties": [ { "id": "custom.displayMode", "value": "color-background" }, { "id": "mappings", "value": [ { "options": { "Failed": { "color": "red", "index": 2 }, "Pending": { "color": "yellow", "index": 0 }, "Unknown": { "color": "red", "index": 1 } }, "type": "value" } ] } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 3 }, "id": 56, "options": { "showHeader": true }, "pluginVersion": "8.0.7", "targets": [ { "exemplar": true, "expr": "sum(kube_pod_status_phase{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\".*$namespace\"}) by (pod, phase)", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "TiDB Cluster Pod Status", "transformations": [ { "id": "filterByValue", "options": { "filters": [ { "config": { "id": "equal", "options": { "value": "0" } }, "fieldName": "Value" } ], "match": "any", "type": "exclude" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true }, "indexByName": { "Time": 0, "Value": 3, "phase": 2, "pod": 1 }, "renameByName": { "phase": "status" } } } ], "type": "table" }, { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null } ] }, "unit": "dtdhms" }, "overrides": [] }, "gridPos": { "h": 8, "w": 4, "x": 12, "y": 3 }, "id": 88, "options": { "colorMode": "value", "graphMode": "area", "justifyMode": "auto", "orientation": "auto", "reduceOptions": { "calcs": ["lastNotNull"], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, "pluginVersion": "8.0.7", "targets": [ { "exemplar": true, "expr": "time() - dispatcher_cluster_creation_time{tidb=\"$cluster_id\"}", "format": "time_series", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "UpTime", "type": "stat" }, { "datasource": null, "description": "Whether tidb cluster is active over the past week", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto" }, "mappings": [ { "options": { "0": { "color": "green", "index": 0, "text": "Inactive" } }, "type": "value" }, { "options": { "from": 0, "result": { "color": "blue", "index": 1, "text": "Active" }, "to": 10000000 }, "type": "range" } ], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "Status" }, "properties": [ { "id": "custom.align", "value": "center" }, { "id": "custom.displayMode", "value": "color-background" } ] } ] }, "gridPos": { "h": 8, "w": 8, "x": 16, "y": 3 }, "id": 62, "options": { "showHeader": true }, "pluginVersion": "8.0.7", "targets": [ { "exemplar": true, "expr": "sum(increase(tidb_executor_statement_total{cluster_id=\"$cluster_id\"}[1d]))[7d:1d]", "format": "table", "instant": true, "interval": "", "legendFormat": "Status", "refId": "A" }, { "exemplar": true, "expr": "sum(increase(tidb_executor_statement_total{cluster_id=\"$cluster_id\"}[1d]))", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "Status", "refId": "B" } ], "title": "Status Over Past Week", "transformations": [ { "id": "seriesToRows", "options": {} }, { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Metric": true }, "indexByName": {}, "renameByName": { "Value": "Status", "Value #B": "Status" } } } ], "type": "table" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "Failed job and long running job", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 11 }, "hiddenSeries": false, "id": 57, "legend": { "alignAsTable": true, "avg": false, "current": false, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(kube_job_status_active{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\".*$namespace\"}) by (job_name)", "hide": false, "interval": "", "legendFormat": "{{job_name}}: Active", "refId": "B" }, { "exemplar": true, "expr": "sum(kube_job_status_failed{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\".*$namespace\"}) by (job_name)", "hide": false, "interval": "", "legendFormat": "{{job_name}}: Failed", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Job Abnormal", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:3215", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:3216", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 11 }, "hiddenSeries": false, "id": 67, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(kube_statefulset_replicas{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\".*$namespace\"}) by (statefulset) - sum(kube_statefulset_status_replicas_updated{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\".*$namespace\"}) by (statefulset)", "interval": "", "legendFormat": "{{statefulset}}: outdated", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "StatefulSet Outdated", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2228", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2229", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 19 }, "hiddenSeries": false, "id": 19, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [ { "targetBlank": true, "title": "Resources Usage Detail Under Current Namespace", "url": "./d/85a562078cdf77779eaa1add43ccec1e/kubernetes-compute-resources-namespace-pods?orgId=1&refresh=10s&var-datasource=default&var-cluster=freetier/eks/${cluster_k8s}&var-namespace=free-tier-${namespace}" } ], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(container_memory_working_set_bytes{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\".*$namespace\", container!=\"\", image!=\"\"}) by (pod)", "interval": "", "legendFormat": "{{pod}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:694", "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:695", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "", "fieldConfig": { "defaults": { "unit": "percentunit" }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 19 }, "hiddenSeries": false, "id": 23, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 300, "total": false, "values": true }, "lines": true, "linewidth": 1, "links": [ { "targetBlank": true, "title": "Resources Usage Detail Under Current Namespace", "url": "./d/85a562078cdf77779eaa1add43ccec1e/kubernetes-compute-resources-namespace-pods?orgId=1&refresh=10s&var-datasource=default&var-cluster=freetier/eks/${cluster_k8s}&var-namespace=free-tier-${namespace}" } ], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\".*$namespace\"}) by (pod)", "interval": "", "legendFormat": "{{pod}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "percentunit", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "links": [ { "targetBlank": true, "title": "Volume Usage Detail", "url": "./d/919b92a8e8041bd567af9edab12c840c/kubernetes-persistent-volumes?orgId=1&refresh=10s&var-datasource=default&var-cluster=freetier/eks/${cluster_k8s}&var-namespace=free-tier-${namespace}&var-volume=${__field.labels.persistentvolumeclaim}" } ], "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 27 }, "id": 70, "options": { "legend": { "calcs": ["min", "max", "mean", "lastNotNull"], "displayMode": "table", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "(\n kubelet_volume_stats_capacity_bytes{cluster=\"freetier/eks/$cluster_k8s\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=~\".*$namespace\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"freetier/eks/$cluster_k8s\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=~\".*$namespace\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"freetier/eks/$cluster_k8s\", job=\"kubelet\", metrics_path=\"/metrics\", namespace=~\".*$namespace\"}\n\n", "hide": false, "interval": "", "legendFormat": "{{persistentvolumeclaim}}", "refId": "A" } ], "title": "PV Space Usage", "type": "timeseries" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "description": "", "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 27 }, "hiddenSeries": false, "id": 72, "legend": { "avg": false, "current": false, "hideZero": true, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "links": [ { "targetBlank": true, "title": "Network Status Under Current Namespace", "url": "./d/8b7a8b326d7a6f1f04244066368c67af/kubernetes-networking-namespace-pods?orgId=1&var-datasource=default&var-namespace=free-tier-${namespace}&var-resolution=5m" } ], "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.7", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(irate(container_network_receive_packets_dropped_total{namespace=~\".*$namespace\"}[$__rate_interval])) by (pod)", "interval": "", "legendFormat": "Receive: {{pod}}", "refId": "A" }, { "exemplar": true, "expr": "sum(irate(container_network_transmit_packets_dropped_total{namespace=~\".*$namespace\"}[$__rate_interval])) by (pod)", "hide": false, "interval": "", "legendFormat": "Transmit: {{pod}}", "refId": "B" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Pod Network Packets Dropped", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "TiDB Cluster", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 19 }, "id": 50, "panels": [ { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "filterable": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "status" }, "properties": [ { "id": "custom.displayMode", "value": "color-background" }, { "id": "mappings", "value": [ { "options": { "Failed": { "color": "red", "index": 0 }, "Pending": { "color": "yellow", "index": 2 }, "Unknown": { "color": "red", "index": 1 } }, "type": "value" } ] } ] }, { "matcher": { "id": "byName", "options": "namespace" }, "properties": [ { "id": "links", "value": [ { "targetBlank": true, "title": "Resources Usage Details Under This Namespace", "url": "./d/85a562078cdf77779eaa1add43ccec1e/kubernetes-compute-resources-namespace-pods?orgId=1&refresh=10s&var-datasource=default&var-cluster=freetier/eks/${cluster_k8s}&var-namespace=${__value.text}" } ] } ] } ] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 4 }, "id": 54, "options": { "showHeader": true, "sortBy": [ { "desc": true, "displayName": "namespace" } ] }, "pluginVersion": "8.0.2", "targets": [ { "exemplar": true, "expr": "sum(kube_pod_status_phase{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (namespace,pod, phase)", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "System Pod Status", "transformations": [ { "id": "filterByValue", "options": { "filters": [ { "config": { "id": "equal", "options": { "value": "0" } }, "fieldName": "Value" } ], "match": "any", "type": "exclude" } }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true }, "indexByName": { "Time": 0, "Value": 4, "namespace": 1, "phase": 3, "pod": 2 }, "renameByName": { "phase": "status" } } } ], "type": "table" }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 4 }, "hiddenSeries": false, "id": 46, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(kube_statefulset_replicas{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (statefulset) - sum(kube_statefulset_status_replicas_updated{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (statefulset)", "interval": "", "legendFormat": "{{statefulset}}: outdated", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "StatefulSet Outdated", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2228", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2229", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 12 }, "hiddenSeries": false, "id": 48, "legend": { "avg": false, "current": false, "hideZero": true, "max": false, "min": false, "show": true, "total": false, "values": false }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(kube_daemonset_status_desired_number_scheduled{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (daemonset) - sum(kube_daemonset_status_number_ready{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (daemonset)", "interval": "", "legendFormat": "{{daemonset}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "DaemonSet Unready", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2395", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2396", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 12 }, "hiddenSeries": false, "id": 45, "legend": { "alignAsTable": true, "avg": false, "current": true, "hideZero": true, "max": false, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(kube_deployment_spec_replicas{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (deployment) - sum(kube_deployment_status_replicas_updated{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (deployment)", "interval": "", "legendFormat": "{{deployment}}: outdated", "refId": "A" }, { "exemplar": true, "expr": "sum(kube_deployment_spec_replicas{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (deployment)", "hide": true, "interval": "", "legendFormat": "{{deployment}}: desired", "refId": "B" }, { "exemplar": true, "expr": "sum(kube_deployment_spec_replicas{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (deployment) - sum(kube_deployment_status_replicas_updated{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (deployment)", "hide": false, "interval": "", "legendFormat": "{{deployment}}: outdated", "refId": "C" }, { "exemplar": true, "expr": "sum(kube_deployment_spec_replicas{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (deployment)", "hide": true, "interval": "", "legendFormat": "{{deployment}}: desired", "refId": "D" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Deployment Outdated", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:2228", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:2229", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 20 }, "hiddenSeries": false, "id": 6, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 349, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(container_memory_working_set_bytes{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\", container!=\"\", image!=\"\"}) by (namespace,pod)", "hide": false, "interval": "", "legendFormat": "{{namespace}}/{{pod}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Memory Usage", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:94", "format": "bytes", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:95", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 20 }, "hiddenSeries": false, "id": 8, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": false, "min": false, "rightSide": true, "show": true, "sideWidth": 350, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (namespace,pod)", "interval": "", "legendFormat": "{{namespace}}/{{pod}}", "refId": "A" } ], "thresholds": [ { "$$hashKey": "object:2571", "colorMode": "critical", "fill": true, "line": true, "op": "gt", "value": 0.25, "yaxis": "left" } ], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "CPU Usage", "tooltip": { "shared": true, "sort": 2, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:418", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:419", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "System Components", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 20 }, "id": 74, "panels": [ { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 5, "w": 24, "x": 0, "y": 5 }, "id": 64, "options": { "legend": { "calcs": ["lastNotNull"], "displayMode": "table", "placement": "right" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "sum(rate(rest_client_requests_total{cluster=\"freetier/eks/$cluster_k8s\",job=\"manager\",code=~\"2..\"}[5m])) by (host)", "interval": "", "legendFormat": "{{host}}:2xx", "refId": "A" }, { "exemplar": true, "expr": "sum(rate(rest_client_requests_total{cluster=\"freetier/eks/$cluster_k8s\",job=\"manager\",code=~\"3..\"}[5m])) by (host)", "hide": false, "interval": "", "legendFormat": "{{host}}:3xx", "refId": "B" }, { "exemplar": true, "expr": "sum(rate(rest_client_requests_total{cluster=\"freetier/eks/$cluster_k8s\",job=\"manager\",code=~\"4..\"}[5m])) by (host)", "hide": false, "interval": "", "legendFormat": "{{host}}:4xx", "refId": "C" }, { "exemplar": true, "expr": "sum(rate(rest_client_requests_total{cluster=\"freetier/eks/$cluster_k8s\",job=\"manager\",code=~\"5..\"}[5m])) by (host)", "hide": false, "interval": "", "legendFormat": "{{host}}:5xx", "refId": "D" } ], "title": "Manager-> Kube API Request Rate", "type": "timeseries" }, { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "left", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "dtdurations" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 10 }, "id": 66, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{cluster=\"freetier/eks/$cluster_k8s\",job=\"manager\",verb=\"POST\"}[5m])) by (verb, url, le))", "interval": "", "legendFormat": "{{url}}", "refId": "A" } ], "title": "POST Request Latency 99th Quantile", "type": "timeseries" }, { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "left", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "dtdurations" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 10 }, "id": 75, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(rest_client_request_latency_seconds_bucket{cluster=\"freetier/eks/$cluster_k8s\",job=\"manager\",verb=~\"GET\"}[5m])) by (verb, url, le))", "interval": "", "legendFormat": "{{url}}", "refId": "A" } ], "title": "GET Request Latency 99th Quantile", "type": "timeseries" }, { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "s" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 18 }, "id": 77, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(workqueue_queue_duration_seconds_bucket{cluster=\"freetier/eks/$cluster_k8s\",job=\"manager\"}[5m])) by (instance, name, le))", "interval": "", "legendFormat": "{{instance}} {{name}}", "refId": "A" } ], "title": "Work Queue 99th Latency", "type": "timeseries" }, { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] }, "unit": "none" }, "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 18 }, "id": 78, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, "tooltip": { "mode": "single" } }, "targets": [ { "exemplar": true, "expr": "workqueue_depth{cluster=\"freetier/eks/$cluster_k8s\",job=\"manager\"}", "interval": "", "legendFormat": "{{instance}} {{name}}", "refId": "A" } ], "title": "Work Queue Depth", "type": "timeseries" } ], "title": "Infra Provider", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 21 }, "id": 12, "panels": [ { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "filterable": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "pod" }, "properties": [] }, { "matcher": { "id": "byName", "options": "pod_ip" }, "properties": [] }, { "matcher": { "id": "byName", "options": "node" }, "properties": [] } ] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 6 }, "id": 59, "options": { "showHeader": true, "sortBy": [] }, "pluginVersion": "8.0.2", "targets": [ { "exemplar": true, "expr": "sum(kube_pod_info{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\".*$namespace\"}) by (pod, pod_ip, node)", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "TiDB Cluster", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true, "__name__": true, "cluster": true, "created_by_kind": true, "created_by_name": true, "host_ip": true, "ignoreAlerts": true, "instance": true, "job": true, "name": true, "namespace": true, "priority_class": true, "prometheus": true, "provider": true, "receive": true, "region": true, "replica": true, "seed_api": true, "seed_provider": true, "seed_region": true, "shoot_infra": true, "shoot_name": true, "tenant": true, "tenant_id": true, "type": true, "uid": true }, "indexByName": { "node": 2, "pod": 0, "pod_ip": 1 }, "renameByName": { "pod_ip": "", "project": "env" } } } ], "type": "table" }, { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "filterable": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "pod" }, "properties": [] }, { "matcher": { "id": "byName", "options": "pod_ip" }, "properties": [] }, { "matcher": { "id": "byName", "options": "node" }, "properties": [] } ] }, "gridPos": { "h": 8, "w": 24, "x": 0, "y": 14 }, "id": 58, "options": { "showHeader": true, "sortBy": [] }, "pluginVersion": "8.0.2", "targets": [ { "exemplar": true, "expr": "sum(kube_pod_info{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (namespace, pod, pod_ip, node)", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "A" } ], "title": "Controlplane", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true, "__name__": true, "cluster": true, "created_by_kind": true, "created_by_name": true, "host_ip": true, "ignoreAlerts": true, "instance": true, "job": true, "name": true, "namespace": false, "priority_class": true, "prometheus": true, "provider": true, "receive": true, "region": true, "replica": true, "seed_api": true, "seed_provider": true, "seed_region": true, "shoot_infra": true, "shoot_name": true, "tenant": true, "tenant_id": true, "type": true, "uid": true }, "indexByName": { "Time": 4, "Value": 5, "namespace": 0, "node": 3, "pod": 1, "pod_ip": 2 }, "renameByName": { "pod_ip": "", "project": "env" } } } ], "type": "table" }, { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "filterable": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [ { "matcher": { "id": "byName", "options": "node" }, "properties": [ { "id": "links", "value": [ { "targetBlank": true, "title": "node details", "url": "./d/fa49a4706d07a042595b664c87fb33ea/nodes?orgId=1&var-datasource=default&var-instance=${__value.text}" } ] } ] }, { "matcher": { "id": "byName", "options": "condition" }, "properties": [ { "id": "custom.displayMode", "value": "color-background" }, { "id": "color", "value": { "fixedColor": "yellow", "mode": "fixed" } }, { "id": "mappings", "value": [ { "options": { "Ready": { "color": "green", "index": 0 } }, "type": "value" } ] } ] } ] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 22 }, "id": 2, "options": { "showHeader": true, "sortBy": [] }, "pluginVersion": "8.0.2", "targets": [ { "exemplar": true, "expr": "sum(kube_node_labels{cluster=\"freetier/eks/$cluster_k8s\"} * on(node) group_left(kubelet_version,kubeproxy_version,container_runtime_version) kube_node_info{cluster=\"freetier/eks/$cluster_k8s\"}) by (node,label_kubernetes_io_hostname,label_failure_domain_beta_kubernetes_io_zone,kubelet_version,kubeproxy_version,container_runtime_version,label_beta_kubernetes_io_instance_type)", "format": "table", "instant": true, "interval": "", "legendFormat": "", "refId": "A" }, { "exemplar": true, "expr": "sum by (node,condition) (kube_node_status_condition{cluster=\"freetier/eks/$cluster_k8s\",status=\"true\"}==1)", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "{{node}}", "refId": "B" } ], "title": "Worker Nodes", "transformations": [ { "id": "merge", "options": {} }, { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true, "Value #A": true, "Value #B": true, "__name__": true, "cluster": true, "ignoreAlerts": true, "instance": true, "job": true, "label_beta_kubernetes_io_arch": true, "label_beta_kubernetes_io_os": true, "label_billing_cluster_id": true, "label_billing_cluster_status_from_central": true, "label_billing_namespace": true, "label_billing_tenant_id": true, "label_cloud_pingcap_com_owner": true, "label_cluster": true, "label_component": true, "label_env": true, "label_failure_domain_beta_kubernetes_io_region": true, "label_gardener_node_az_info": true, "label_gardener_node_machine_type": true, "label_gardener_wg_name": false, "label_gardener_wg_prefix_name": true, "label_kubernetes_io_arch": true, "label_kubernetes_io_os": true, "label_node_kubernetes_io_instance_type": true, "label_node_kubernetes_io_role": true, "label_pingcap_com_aws_local_ssd": true, "label_project": true, "label_tenant": true, "label_topology_ebs_csi_aws_com_zone": true, "label_topology_kubernetes_io_region": true, "label_topology_kubernetes_io_zone": true, "label_worker_garden_sapcloud_io_group": true, "name": true, "node": true, "receive": true, "replica": true, "seed_api": true, "seed_provider": true, "seed_region": true, "shoot_infra": true, "shoot_name": true, "tenant": true, "tenant_id": true, "type": true }, "indexByName": { "Time": 13, "Value": 40, "__name__": 14, "cluster": 15, "ignoreAlerts": 16, "instance": 17, "job": 18, "label_beta_kubernetes_io_arch": 19, "label_beta_kubernetes_io_instance_type": 2, "label_beta_kubernetes_io_os": 20, "label_cloud_pingcap_com_owner": 6, "label_component": 21, "label_env": 22, "label_failure_domain_beta_kubernetes_io_region": 23, "label_failure_domain_beta_kubernetes_io_zone": 1, "label_gardener_node_az_info": 24, "label_gardener_node_machine_type": 25, "label_gardener_wg_name": 4, "label_gardener_wg_prefix_name": 7, "label_kubernetes_io_arch": 26, "label_kubernetes_io_hostname": 0, "label_kubernetes_io_os": 27, "label_node_kubernetes_io_role": 28, "label_pingcap_com_aws_local_ssd": 8, "label_project": 9, "label_tenant": 29, "label_worker_garden_sapcloud_io_group": 10, "label_worker_gardener_cloud_pool": 3, "name": 30, "node": 31, "project": 5, "receive": 32, "replica": 33, "seed_api": 34, "seed_provider": 35, "seed_region": 36, "shoot_infra": 37, "shoot_name": 11, "tenant": 12, "tenant_id": 38, "type": 39 }, "renameByName": { "label_beta_kubernetes_io_instance_type": "instance type", "label_cloud_pingcap_com_owner": "", "label_failure_domain_beta_kubernetes_io_zone": "zone", "label_gardener_wg_name": "used by", "label_gardener_wg_prefix_name": "", "label_kubernetes_io_hostname": "node", "label_pingcap_com_aws_local_ssd": "", "label_project": "project", "label_worker_garden_sapcloud_io_group": "", "label_worker_gardener_cloud_pool": "worker group", "project": "env", "shoot_name": "" } } } ], "type": "table" } ], "title": "Node & Pod Mapping", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 22 }, "id": 80, "panels": [ { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "links": [ { "targetBlank": true, "title": "Node Usage Detail", "url": "./d/fa49a4706d07a042595b664c87fb33ea/nodes?orgId=1&var-datasource=default&var-instance=${__field.labels.instance}" } ] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 23 }, "hiddenSeries": false, "id": 82, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "avg(label_replace(node_load15{ cluster=~\"freetier/eks/$cluster_k8s\"},\"cluster\",\"$1\",\"cluster\",\"freetier/eks/(.*)\")) by (cluster,instance)", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nodes Load 15", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:172", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:173", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fieldConfig": { "defaults": { "links": [ { "targetBlank": true, "title": "Node Usage Detail", "url": "./d/fa49a4706d07a042595b664c87fb33ea/nodes?orgId=1&var-datasource=default&var-instance=${__field.labels.instance}" } ] }, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 23 }, "hiddenSeries": false, "id": 83, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "avg(label_replace(node_load5{ cluster=~\"freetier/eks/$cluster_k8s\"},\"cluster\",\"$1\",\"cluster\",\"freetier/eks/(.*)\")) by (cluster,instance)", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Nodes Load 5", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:172", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:173", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 0, "y": 31 }, "hiddenSeries": false, "id": 85, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "label_replace(avg(1 - avg without (cpu, mode) (\n rate(node_cpu_seconds_total{job=\"node-exporter\", mode=\"idle\", cluster=~\"freetier/eks/$cluster_k8s\"}[$__rate_interval]))) by (cluster,instance),\"cluster\",\"$1\",\"cluster\",\"freetier/eks/(.*)\")", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Node CPU Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:343", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:344", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, "datasource": null, "fill": 1, "fillGradient": 0, "gridPos": { "h": 8, "w": 12, "x": 12, "y": 31 }, "hiddenSeries": false, "id": 86, "legend": { "alignAsTable": true, "avg": false, "current": true, "max": true, "min": false, "rightSide": true, "show": true, "total": false, "values": true }, "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, "pluginVersion": "8.0.2", "pointradius": 2, "points": false, "renderer": "flot", "seriesOverrides": [], "spaceLength": 10, "stack": false, "steppedLine": false, "targets": [ { "exemplar": true, "expr": "label_replace(\n sum(1 - (\n sum(node_memory_MemAvailable_bytes{job=\"node-exporter\", cluster=~\"freetier/eks/$cluster_k8s\"}) by (cluster,instance))\n /\n sum( node_memory_MemTotal_bytes{job=\"node-exporter\", cluster=~\"freetier/eks/$cluster_k8s\"} ) by (cluster,instance)\n ) by (cluster,instance)\n ,\"cluster\",\"$1\",\"cluster\",\"freetier/eks/(.*)\"\n)", "interval": "", "legendFormat": "{{instance}}", "refId": "A" } ], "thresholds": [], "timeFrom": null, "timeRegions": [], "timeShift": null, "title": "Node Memory Usage", "tooltip": { "shared": true, "sort": 0, "value_type": "individual" }, "type": "graph", "xaxis": { "buckets": null, "mode": "time", "name": null, "show": true, "values": [] }, "yaxes": [ { "$$hashKey": "object:343", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true }, { "$$hashKey": "object:344", "format": "short", "label": null, "logBase": 1, "max": null, "min": null, "show": true } ], "yaxis": { "align": false, "alignLevel": null } } ], "title": "Node Resources", "type": "row" }, { "collapsed": true, "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, "id": 14, "panels": [ { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "filterable": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 24 }, "id": 60, "options": { "frameIndex": 1, "showHeader": true }, "pluginVersion": "8.0.2", "targets": [ { "exemplar": true, "expr": "sum(kube_pod_container_info{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\".*$namespace\"}) by (pod, container, image, image_id)", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "refId": "B" } ], "title": "TiDB Cluster Pod Image", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true, "__name__": true, "cluster": true, "container_id": true, "ignoreAlerts": true, "instance": true, "job": true, "name": true, "namespace": true, "project": true, "prometheus": true, "provider": true, "receive": true, "region": true, "replica": true, "seed_api": true, "seed_provider": true, "seed_region": true, "shoot_infra": true, "shoot_name": true, "tenant": true, "tenant_id": true, "type": true }, "indexByName": { "Time": 2, "Value": 17, "__name__": 3, "cluster": 4, "container": 5, "container_id": 6, "image": 7, "image_id": 8, "instance": 9, "job": 10, "namespace": 0, "pod": 1, "prometheus": 11, "provider": 12, "receive": 13, "region": 14, "replica": 15, "tenant_id": 16 }, "renameByName": {} } } ], "type": "table" }, { "datasource": null, "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "custom": { "align": "auto", "displayMode": "auto", "filterable": true }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { "color": "green", "value": null }, { "color": "red", "value": 80 } ] } }, "overrides": [] }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 31 }, "id": 68, "options": { "frameIndex": 1, "showHeader": true, "sortBy": [ { "desc": false, "displayName": "namespace" } ] }, "pluginVersion": "8.0.2", "targets": [ { "exemplar": true, "expr": "sum(kube_pod_container_info{cluster=\"freetier/eks/$cluster_k8s\", namespace=~\"$system_ns\"}) by (namespace, pod, container, image, image_id)", "format": "table", "hide": false, "instant": true, "interval": "", "legendFormat": "", "refId": "B" } ], "title": "System Pods Image", "transformations": [ { "id": "organize", "options": { "excludeByName": { "Time": true, "Value": true, "__name__": true, "cluster": true, "container_id": true, "ignoreAlerts": true, "instance": true, "job": true, "name": true, "namespace": false, "project": true, "prometheus": true, "provider": true, "receive": true, "region": true, "replica": true, "seed_api": true, "seed_provider": true, "seed_region": true, "shoot_infra": true, "shoot_name": true, "tenant": true, "tenant_id": true, "type": true }, "indexByName": { "Time": 2, "Value": 17, "__name__": 3, "cluster": 4, "container": 5, "container_id": 6, "image": 7, "image_id": 8, "instance": 9, "job": 10, "namespace": 0, "pod": 1, "prometheus": 11, "provider": 12, "receive": 13, "region": 14, "replica": 15, "tenant_id": 16 }, "renameByName": {} } } ], "type": "table" } ], "title": "Image", "type": "row" } ], "refresh": "", "schemaVersion": 30, "style": "dark", "tags": ["Dev-tier"], "templating": { "list": [ { "allValue": null, "current": { "selected": true, "text": "prod-ap-northeast-1-f01", "value": "prod-ap-northeast-1-f01" }, "datasource": null, "definition": "label_values(dispatcher_cluster_status_phase, kubernetes)", "description": "EKS Cluster Name", "error": null, "hide": 0, "includeAll": false, "label": "EKS Cluster", "multi": false, "name": "cluster_k8s", "options": [], "query": { "query": "label_values(dispatcher_cluster_status_phase, kubernetes)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allValue": null, "current": { "selected": true, "text": "1379661944583944071", "value": "1379661944583944071" }, "datasource": null, "definition": "label_values(dispatcher_cluster_status_phase{kubernetes=\"$cluster_k8s\"},tidb)", "description": "TiDB Cluster ID", "error": null, "hide": 0, "includeAll": false, "label": "TiDB Cluster", "multi": false, "name": "cluster_id", "options": [], "query": { "query": "label_values(dispatcher_cluster_status_phase{kubernetes=\"$cluster_k8s\"},tidb)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allValue": null, "current": { "selected": false, "text": "cloud-tidb-cluster", "value": "cloud-tidb-cluster" }, "datasource": null, "definition": "label_values(dbaas_tidb_cluster_info{cluster_id=\"$cluster_id\"},name)", "description": "TiDB Cluster name", "error": null, "hide": 1, "includeAll": false, "label": "", "multi": false, "name": "cluster", "options": [], "query": { "query": "label_values(dbaas_tidb_cluster_info{cluster_id=\"$cluster_id\"},name)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allValue": null, "current": { "selected": false, "text": "pingcap", "value": "pingcap" }, "datasource": null, "definition": "label_values(dbaas_tidb_cluster_info{cluster_id=\"$cluster_id\"},tenant_name)", "description": "Tenant Name", "error": null, "hide": 0, "includeAll": false, "label": "Tenant", "multi": false, "name": "tenant_name", "options": [], "query": { "query": "label_values(dbaas_tidb_cluster_info{cluster_id=\"$cluster_id\"},tenant_name)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allValue": null, "current": { "selected": false, "text": "1372813089188361294", "value": "1372813089188361294" }, "datasource": null, "definition": "label_values(dbaas_tidb_cluster_info{cluster_id=\"$cluster_id\"},tenant)", "description": "Tenant ID", "error": null, "hide": 1, "includeAll": false, "label": "", "multi": false, "name": "tenant_id", "options": [], "query": { "query": "label_values(dbaas_tidb_cluster_info{cluster_id=\"$cluster_id\"},tenant)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allValue": null, "current": { "selected": false, "text": "default project", "value": "default project" }, "datasource": null, "definition": "label_values(dbaas_tidb_cluster_info{cluster_id=\"$cluster_id\"},project_name)", "description": "Project Name", "error": null, "hide": 0, "includeAll": false, "label": "Project", "multi": false, "name": "project_name", "options": [], "query": { "query": "label_values(dbaas_tidb_cluster_info{cluster_id=\"$cluster_id\"},project_name)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "allValue": null, "current": { "selected": false, "text": "prod-1372813089188451286", "value": "prod-1372813089188451286" }, "datasource": null, "definition": "label_values(dispatcher_cluster_status_phase{tidb=\"$cluster_id\"},namespace)", "description": "Project Namespace", "error": null, "hide": 1, "includeAll": false, "label": "", "multi": false, "name": "namespace", "options": [], "query": { "query": "label_values(dispatcher_cluster_status_phase{tidb=\"$cluster_id\"},namespace)", "refId": "StandardVariableQuery" }, "refresh": 1, "regex": "", "skipUrlSync": false, "sort": 0, "type": "query" }, { "description": null, "error": null, "hide": 2, "label": null, "name": "system_ns", "query": "tidb-admin|kube-system|cert-manager|monitoring|logging|calico-system|tigera-operator", "skipUrlSync": false, "type": "constant" } ] }, "time": { "from": "now-6h", "to": "now" }, "timepicker": { "hidden": false }, "timezone": "", "title": "Dev Tier Troubleshooting", "uid": "p1FdsDc7z", "version": 1 }