{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "-- Grafana --",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "type": "dashboard"
      }
    ]
  },
  "editable": true,
  "gnetId": null,
  "graphTooltip": 0,
  "id": 285,
  "links": [],
  "panels": [
    {
      "datasource": "tidb-cluster",
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "thresholds"
          },
          "custom": {
            "align": "auto",
            "displayMode": "auto",
            "filterable": true
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "string"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 0
      },
      "id": 15,
      "options": {
        "showHeader": true
      },
      "pluginVersion": "8.0.7",
      "targets": [
        {
          "exemplar": true,
          "expr": "sum(label_replace(infra_api_resources_cluster_info{},\"cluster_id\",\"$1\",\"name\",\"(.*)\"))by(cluster_id,status)>=0",
          "format": "table",
          "instant": true,
          "interval": "",
          "legendFormat": "",
          "refId": "A"
        }
      ],
      "title": "Cluster Status(offline is unavailable)",
      "transformations": [
        {
          "id": "organize",
          "options": {
            "excludeByName": {
              "Time": true,
              "Value": true,
              "Value #A": true
            },
            "indexByName": {},
            "renameByName": {}
          }
        }
      ],
      "type": "table"
    },
    {
      "datasource": null,
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "line+area"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0.7
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 0
      },
      "id": 6,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "exemplar": true,
          "expr": "1- label_replace(\n      sum(:node_memory_MemAvailable_bytes:kube_node_labels{label_gardener_wg_prefix_name=~\".*tikv\"})by(label_gardener_wg_prefix_name)\n      /\n      sum(:node_memory_MemTotal_bytes:kube_node_labels{label_gardener_wg_prefix_name=~\".*tikv\"})by(label_gardener_wg_prefix_name),\n      \"cluster_id\",\"$2\", \"label_gardener_wg_prefix_name\", \"(tidb)?(.+)-.*\")",
          "interval": "",
          "legendFormat": "{{ cluster_id }}",
          "refId": "A"
        }
      ],
      "title": "Total TiKV memory usage",
      "type": "timeseries"
    },
    {
      "datasource": null,
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "line+area"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0.7
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 8
      },
      "id": 4,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "exemplar": true,
          "expr": "1- label_replace(\n      sum(:node_memory_MemAvailable_bytes:kube_node_labels{label_gardener_wg_prefix_name=~\".*tidb\",label_gardener_wg_prefix_name!~\".*extra.*\"})by(label_gardener_wg_prefix_name)\n      /\n      sum(:node_memory_MemTotal_bytes:kube_node_labels{label_gardener_wg_prefix_name=~\".*tidb\",label_gardener_wg_prefix_name!~\".*extra.*\"})by(label_gardener_wg_prefix_name),\n      \"cluster_id\",\"$2\", \"label_gardener_wg_prefix_name\", \"(tidb)?(.+)-.*\")",
          "interval": "",
          "legendFormat": "{{ cluster_id }}",
          "refId": "A"
        }
      ],
      "title": "Total TiDB memory usage",
      "type": "timeseries"
    },
    {
      "datasource": null,
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "line+area"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 1
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 8
      },
      "id": 7,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "exemplar": true,
          "expr": "label_replace(increase(kube_pod_container_status_restarts_total{cluster=~\"(shoot|staging|prod).*\",container=\"tidb\"}[5m]) >= 1 AND ignoring(reason) kube_pod_container_status_last_terminated_reason{cluster=~\"(shoot|staging|prod).*\",reason='OOMKilled',container=\"tidb\"} == 1,\"cluster_id\",\"$2\", \"namespace\", \"(tidb)(.*)\")",
          "hide": false,
          "interval": "",
          "legendFormat": "{{ cluster_id }}",
          "refId": "A"
        },
        {
          "exemplar": true,
          "expr": "sum(label_replace(increase(kube_pod_container_status_restarts_total{cluster=~\"freetier.*\",container=\"tidb\"}[5m])>=1\n      AND ignoring(reason) kube_pod_container_status_last_terminated_reason{cluster=~\"freetier.*\",reason='OOMKilled',container=\"tidb\"}==1,\"cluster_id\",\"$1\",\"pod\",\"db(.*)-(.*)-(.*)\"))by(cluster_id)",
          "hide": false,
          "interval": "",
          "legendFormat": "{{ cluster_id }}",
          "refId": "B"
        }
      ],
      "title": "TiDB OOM History",
      "type": "timeseries"
    },
    {
      "datasource": null,
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "line+area"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0.7
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 16
      },
      "id": 5,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "exemplar": true,
          "expr": "1- label_replace(\n      sum(:node_memory_MemAvailable_bytes:kube_node_labels{label_gardener_wg_prefix_name=~\".*tiflash\"})by(label_gardener_wg_prefix_name)\n      /\n      sum(:node_memory_MemTotal_bytes:kube_node_labels{label_gardener_wg_prefix_name=~\".*tiflash\"})by(label_gardener_wg_prefix_name),\n      \"cluster_id\",\"$2\", \"label_gardener_wg_prefix_name\", \"(tidb)?(.+)-.*\")",
          "interval": "",
          "legendFormat": "{{ cluster_id }}",
          "refId": "A"
        }
      ],
      "title": "Total TiFlash memory usage",
      "type": "timeseries"
    },
    {
      "datasource": null,
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "line+area"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0.8
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 16
      },
      "id": 9,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "exemplar": true,
          "expr": "label_replace(avg(1-:node_cpu_idle:kube_node_labels{label_gardener_wg_prefix_name=~\".*tikv\"})by(label_gardener_wg_prefix_name),\"cluster_id\",\"$2\", \"label_gardener_wg_prefix_name\", \"(tidb)?(.+)-.*\")",
          "interval": "",
          "legendFormat": "{{ cluster_id }}",
          "refId": "A"
        }
      ],
      "title": "Total TiKV CPU Usage",
      "type": "timeseries"
    },
    {
      "datasource": null,
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "line+area"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0.8
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 24
      },
      "id": 8,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "exemplar": true,
          "expr": "label_replace(avg(1-:node_cpu_idle:kube_node_labels{label_gardener_wg_prefix_name=~\".*tidb\",label_gardener_wg_prefix_name!~\".*extra.*\"})by(label_gardener_wg_prefix_name),\"cluster_id\",\"$2\", \"label_gardener_wg_prefix_name\", \"(tidb)?(.+)(-.*)\")",
          "interval": "",
          "legendFormat": "{{ cluster_id }}",
          "refId": "A"
        }
      ],
      "title": "Total TiDB CPU Usage",
      "type": "timeseries"
    },
    {
      "datasource": null,
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "line+area"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0.8
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 24
      },
      "id": 10,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "exemplar": true,
          "expr": "label_replace(avg(1-:node_cpu_idle:kube_node_labels{label_gardener_wg_prefix_name=~\".*tiflash\"})by(label_gardener_wg_prefix_name),\"cluster_id\",\"$2\", \"label_gardener_wg_prefix_name\", \"(tidb)?(.+)(-.*)\")",
          "interval": "",
          "legendFormat": "{{ cluster_id }}",
          "refId": "A"
        }
      ],
      "title": "Total Tiflash CPU Usage",
      "type": "timeseries"
    },
    {
      "datasource": null,
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "line+area"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0.8
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 0,
        "y": 32
      },
      "id": 12,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "exemplar": true,
          "expr": "1 - label_replace(sum(kubelet_volume_stats_available_bytes{job=\"kubelet\",gardener_wg_prefix_name=~\".*tiflash\"})by(gardener_wg_prefix_name)\n      /\n      sum(kubelet_volume_stats_capacity_bytes{job=\"kubelet\",gardener_wg_prefix_name=~\".*tiflash\"})by(gardener_wg_prefix_name),\"cluster_id\",\"$2\", \"gardener_wg_prefix_name\", \"(tidb)?(.+)(-.*)\")",
          "interval": "",
          "legendFormat": "{{ cluster_id }}-{{persistentvolumeclaim}}",
          "refId": "A"
        },
        {
          "exemplar": true,
          "expr": "1 - sum(label_replace(kubelet_volume_stats_available_bytes{cluster=~\"freetier.*\",job=\"kubelet\",persistentvolumeclaim=~\"tiflash.*\"},\"cluster_id\",\"$1\",\"persistentvolumeclaim\",\"tiflash-db(.*)-tiflash-(.*)\"))by(cluster_id)\n      / \n      sum(label_replace(kubelet_volume_stats_capacity_bytes{cluster=~\"freetier.*\",job=\"kubelet\",persistentvolumeclaim=~\"tiflash.*\"},\"cluster_id\",\"$1\",\"persistentvolumeclaim\",\"tiflash-db(.*)-tiflash-(.*)\"))by(cluster_id)",
          "hide": true,
          "interval": "",
          "legendFormat": "{{ cluster_id }}-{{persistentvolumeclaim}}",
          "refId": "B"
        }
      ],
      "title": "Total TiFlash Storage Usage",
      "type": "timeseries"
    },
    {
      "datasource": null,
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "auto",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "line+area"
            }
          },
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 0.8
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 8,
        "w": 12,
        "x": 12,
        "y": 32
      },
      "id": 11,
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "single"
        }
      },
      "targets": [
        {
          "exemplar": true,
          "expr": "1 - label_replace(sum(kubelet_volume_stats_available_bytes{job=\"kubelet\",gardener_wg_prefix_name=~\".*tikv\"})by(gardener_wg_prefix_name)\n      /\n      sum(kubelet_volume_stats_capacity_bytes{job=\"kubelet\",gardener_wg_prefix_name=~\".*tikv\"})by(gardener_wg_prefix_name),\"cluster_id\",\"$2\", \"gardener_wg_prefix_name\", \"(tidb)?(.+)(-.*)\")",
          "interval": "",
          "legendFormat": "{{ cluster_id }}-{{persistentvolumeclaim}}",
          "refId": "A"
        },
        {
          "exemplar": true,
          "expr": "1 - sum(label_replace(kubelet_volume_stats_available_bytes{cluster=~\"freetier.*\",job=\"kubelet\",persistentvolumeclaim=~\"tikv.*\"},\"cluster_id\",\"$1\",\"persistentvolumeclaim\",\"tikv-db(.*)-tikv-(.*)\"))by(cluster_id)\n      / \n      sum(label_replace(kubelet_volume_stats_capacity_bytes{cluster=~\"freetier.*\",job=\"kubelet\",persistentvolumeclaim=~\"tikv.*\"},\"cluster_id\",\"$1\",\"persistentvolumeclaim\",\"tikv-db(.*)-tikv-(.*)\"))by(cluster_id)",
          "hide": false,
          "interval": "",
          "legendFormat": "{{ cluster_id }}-{{persistentvolumeclaim}}",
          "refId": "B"
        }
      ],
      "title": "Total TiKV Storage Usage",
      "type": "timeseries"
    }
  ],
  "refresh": "",
  "schemaVersion": 30,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": []
  },
  "time": {
    "from": "now-6h",
    "to": "now"
  },
  "timepicker": {},
  "timezone": "",
  "title": "User Alert Dashboard",
  "uid": "Sr8ttpbnk",
  "version": 10
}