groups:
  - name: csng.pod
    rules:
      - alert: CPUUsageIsTooHigh
        annotations:
          message: CPU utilization of pod {{ $labels.pod }} on region {{ $labels.tidb_cluster_id }} is more than 90%.
          diagnosis: 'Click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Ak6VpZD4z%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22code%22,%22expr%22:%22(sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate%7B%20pod%3D%5C%22{{ $labels.pod }}%5C%22%7D)%20by%20(pod)/sum(kube_pod_container_resource_limits%7Bresource%3D%5C%22cpu%5C%22,%20pod%3D%5C%22{{ $labels.pod }}%5C%22%7D)%20by%20(pod))%20*100%22,%22legendFormat%22:%22__auto%22,%22range%22:true,%22instant%22:true,%22hide%22:false%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D'
        expr: |
          (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=~"cluster-ng-service"}) by (namespace,pod)/
            sum(kube_pod_container_resource_limits{resource="cpu",namespace=~"cluster-ng-service"}) by (namespace,pod)
          ) *100 > 90
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: cluster-ng-service
          oncall_team: premium
          tier: next-gen

      - alert: MemUsageIsTooHigh
        annotations:
          message: Memory utilization of pod {{ $labels.pod }} on region {{ $labels.tidb_cluster_id }} is more than 90%.
          diagnosis: 'Click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left=%7B%22datasource%22:%22Ak6VpZD4z%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22editorMode%22:%22code%22,%22expr%22:%22(sum(container_memory_working_set_bytes%7Bcontainer!%3D%5C%22POD%5C%22,%20container!%3D%5C%22%5C%22,%20image!%3D%5C%22%5C%22,%20pod%3D%5C%22{{ $labels.pod }}%5C%22%7D)by(pod)/sum(kube_pod_container_resource_limits%7Bresource%3D%5C%22memory%5C%22,%20pod%3D%5C%22{{ $labels.pod }}%5C%22%7D)%20by%20(pod)%20*%20100)%22,%22legendFormat%22:%22__auto%22,%22range%22:true,%22instant%22:true,%22hide%22:false%7D%5D,%22range%22:%7B%22from%22:%22now-1h%22,%22to%22:%22now%22%7D%7D'
        expr: |
          (sum(container_memory_working_set_bytes{namespace=~"cluster-ng-service",container!="POD", container!="", image!=""})by(pod)/
          sum(kube_pod_container_resource_limits{resource="memory",namespace=~"cluster-ng-service"}) by (pod)
          * 100) > 90
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: cluster-ng-service
          oncall_team: premium
          tier: next-gen

      - alert: PodRestartsFrequently
        annotations:
          summary: The pod {{ $labels.pod }} on region {{ $labels.tidb_cluster_id }} restarts frequently.
          message: Container {{ $labels.namespace }}/{{ $labels.container }} on region {{ $labels.tidb_cluster_id }} is restarting {{ printf "%.2f" $value }} times.
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace="cluster-ng-service"}[2m]) > 2
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: cluster-ng-service
          oncall_team: premium
          tier: next-gen

      - alert: PodIsDown
        annotations:
          summary: The pod {{ $labels.pod }} on region {{ $labels.tidb_cluster_id }} is down
          message: The pod {{ $labels.namespace }}/{{ $labels.pod }} on region {{ $labels.tidb_cluster_id }} is down.
        expr: |
          up{namespace=~"cluster-ng-service"} == 0
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: cluster-ng-service
          oncall_team: premium
          tier: next-gen