groups:
- name: pod.rules
  rules:
  - alert: CPUUsageIsTooHigh
    annotations:
      message: CPU utilization of pod {{ $labels.namespace }} / {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 80% for 3m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-pod=tidb-http-gateway-0&var-container=tidb&var-namespace=tidb-admin&from=now-30m&to=now)
    expr: |
      (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace!~"vip-tidb-pool|import-jobs|kube-system", container!~"cop-worker|tidb-worker-job|tiflash|node-problem-detector.*"}) by (namespace, pod) /
        sum(kube_pod_container_resource_limits{resource="cpu", namespace!~"vip-tidb-pool|import-jobs|kube-system", container!~"cop-worker|tidb-worker-job|tiflash|node-problem-detector.*"}) by (namespace, pod)
      ) * 100 > 80
    for: 3m
    labels:
      severity: warning
      component: k8s-pod
      oncall_team: serverless
  - alert: MemUsageIsTooHigh
    annotations:
      message: Memory utilization of pod {{ $labels.namespace }} / {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 80% for 5m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-pod=tidb-http-gateway-0&var-container=tidb&var-namespace=tidb-admin&from=now-30m&to=now)
    expr: |
      (
        sum by (namespace, pod) (container_memory_rss{namespace!~"cert-manager|tidb-vpa|vip-tidb-pool|kubernetes-dashboard|kube-system|kruise-system", container!="", pod!="", container!~"replication-worker|serverless-meta|cop-worker|tidb-worker-job|tiflash"}) /
        sum by (namespace, pod) (kube_pod_container_resource_limits{resource="memory", namespace!~"cert-manager|tidb-vpa|vip-tidb-pool|kubernetes-dashboard|kube-system|kruise-system", container!~"replication-worker|serverless-meta|cop-worker|tidb-worker-job|tiflash"})
      ) * 100 > 80
      and on(namespace, pod)
      sum by (namespace, pod) (container_memory_working_set_bytes{namespace!~"cert-manager|tidb-vpa|vip-tidb-pool|kubernetes-dashboard|kube-system|kruise-system", container!="", container!="POD", container!~"replication-worker|serverless-meta|cop-worker|tidb-worker-job|tiflash", image!=""}) >
      sum by (namespace, pod) (kube_pod_container_resource_limits{resource="memory", namespace!~"cert-manager|tidb-vpa|vip-tidb-pool|kubernetes-dashboard|kube-system|kruise-system", container!~"replication-worker|serverless-meta|cop-worker|tidb-worker-job|tiflash"}) * 0.8
      and on(namespace, pod)
      sum by (namespace, pod) (container_memory_working_set_bytes{namespace!~"cert-manager|tidb-vpa|vip-tidb-pool|kubernetes-dashboard|kube-system|kruise-system", container!="", container!="POD", container!~"replication-worker|serverless-meta|cop-worker|tidb-worker-job|tiflash", image!=""}) >
      sum by (namespace, pod) (kube_pod_container_resource_limits{resource="memory", namespace!~"cert-manager|tidb-vpa|vip-tidb-pool|kubernetes-dashboard|kube-system|kruise-system", container!~"replication-worker|serverless-meta|cop-worker|tidb-worker-job|tiflash"}) - 10 * 1024 * 1024 * 1024
    for: 5m
    labels:
      severity: warning
      component: k8s-pod
      oncall_team: serverless
  - alert: CPUUsageIsTooHigh
    annotations:
      message: CPU utilization of pod {{ $labels.namespace }} / {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 90% for 5m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-pod=tidb-http-gateway-0&var-container=tidb&var-namespace=tidb-admin&from=now-30m&to=now)
    expr: |
      (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace!~"vip-tidb-pool", container!~"|cop-worker|tidb-worker-job|tiflash|node-problem-detector.*"}) by (namespace, pod) /
        sum(kube_pod_container_resource_limits{resource="cpu", namespace!~"vip-tidb-pool", container!~"|cop-worker|tidb-worker-job|tiflash|node-problem-detector.*"}) by (namespace, pod)
      ) * 100 > 90
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: k8s-pod
      oncall_team: serverless
  - alert: MemUsageIsTooHigh
    annotations:
      message: Memory utilization of pod {{ $labels.namespace }} / {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 90% for 5m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-pod=tidb-http-gateway-0&var-container=tidb&var-namespace=tidb-admin&from=now-30m&to=now)
    expr: |
      ((sum(container_memory_rss{namespace!~"vip-tidb-pool|kubernetes-dashboard|kube-system|kruise-system", container!~"POD|cop-worker|tidb-worker-job|tiflash", image!="",pod!=""}) by (namespace, pod) /
        sum(kube_pod_container_resource_limits{resource="memory", namespace!~"vip-tidb-pool|kubernetes-dashboard|kube-system|kruise-system", container!~"POD|cop-worker|tidb-worker-job|tiflash"}) by (namespace, pod)
      ) * 100 > 90)
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: k8s-pod
      oncall_team: serverless
  - alert: KubePodCrashLooping
    expr: increase(kube_pod_container_status_restarts_total[1h]) > 5
    for: 5m
    labels:
      severity: warning
      provider_type: aws-free-tier
      tier: serverless
      component: k8s-pod
      oncall_team: serverless
    annotations:
      description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container}}) is restarting {{ printf "%.2f" $value }} times in 1 hours. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-pod=tidb-http-gateway-0&var-container=tidb&var-namespace=tidb-admin&from=now-30m&to=now). There are not running pods.'
      summary: 'Pod restarts more than 5 times in 1 hour'
  - alert: KubePodCrashLooping
    expr: increase(kube_pod_container_status_restarts_total{container!~"node-problem-detector|kubernetes-dashboard|kube-eventer|vpa-agent"}[1h]) > 5
    for: 30m
    labels:
      severity: major
      response_class: ticket
      provider_type: aws-free-tier
      tier: serverless
      component: k8s-pod
      oncall_team: serverless
    annotations:
      description: 'Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container}}) is restarting {{ printf "%.2f" $value }} times in 1 hours. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-pod=tidb-http-gateway-0&var-container=tidb&var-namespace=tidb-admin&from=now-30m&to=now). There are not running pods.'
      summary: 'Pod restarts more than 5 times in 1 hour'
  - alert: KubePodPhaseFailed
    annotations:
      message: ns {{ $labels.namespace }} has {{ $value }} number of {{ $labels.phase }} pod for longer than 1 hour. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-pod=tidb-http-gateway-0&var-container=tidb&var-namespace=tidb-admin&from=now-30m&to=now)
    expr: |
      sum(kube_pod_status_phase{phase!~"Succeeded|Running",provider_type=~"alicloud-serverless|aws-free-tier",namespace!="local-path-storage"}) by (namespace,phase) > 1
    for: 1h
    labels:
      severity: warning
      provider_type: aws-free-tier
      tier: serverless
      component: k8s-pod
      oncall_team: serverless
  - alert: PVCUsageTooHigh
    annotations:
      message: Disk utilization of {{ $labels.namespace }} / {{ $labels.persistentvolumeclaim }} pvc is more than {{ printf "%.2f" $value }}% for 10m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=ap-southeast-1-f01%2Faws-ap-southeast-1%2Fdevtier-infra&var-pod=tidb-http-gateway-0&var-container=tidb&var-namespace=tidb-admin&from=now-30m&to=now)
    expr: |
      (1 - kubelet_volume_stats_available_bytes{persistentvolumeclaim!~".*cop-worker.*|.*tiflash.*"}/kubelet_volume_stats_capacity_bytes{persistentvolumeclaim!~".*cop-worker.*|.*tiflash.*"}) * 100 > 80
    for: 10m
    labels:
      tier: serverless
      provider_type: aws-free-tier
      severity: warning
      component: k8s-pod
      oncall_team: serverless
  - alert: LocalPathStoragePodPhaseFailed
    annotations:
      message: The Local Path Storage Provisioner has not been running for longer than one hour.
    expr: |
      sum(kube_pod_status_phase{phase!~"Succeeded|Running",provider_type=~"alicloud-serverless|aws-free-tier",namespace!="local-path-storage",pod=~"local-path-provisioner-.*"}) by (phase) > 0
    for: 1h
    labels:
      severity: warning
      provider_type: aws-free-tier
      tier: serverless
      component: k8s-pod
      oncall_team: serverless