groups:
  - name: mgmt.pod
    rules:
      - alert: CPUHigh
        annotations:
          message: CPU > 80% ns={{ $labels.namespace }} pod={{ $labels.pod }}
        expr: |
          (
            100 *
            (
              sum by (namespace,pod) (
                rate(container_cpu_usage_seconds_total{namespace=~"tidb-management-service",container!="",image!=""}[5m])
              )
              /
              clamp_min(
                sum by (namespace,pod) (
                  kube_pod_container_resource_limits{namespace=~"tidb-management-service",resource="cpu"}
                ),
                0.001
              )
            ) > 80
          )
          and on (namespace,pod)
          (
            max by (namespace,pod) (
              kube_pod_status_phase{namespace=~"tidb-management-service",phase="Running"} == 1
            )
          )
          and on (namespace,pod)
          (
            sum by (namespace,pod) (
              kube_pod_container_resource_limits{namespace=~"tidb-management-service",resource="cpu"}
            ) > 0
          )
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: CPUVeryHigh
        annotations:
          message: CPU > 90% ns={{ $labels.namespace }} pod={{ $labels.pod }}
        expr: |
          (
            100 *
            (
              sum by (namespace,pod) (
                rate(container_cpu_usage_seconds_total{namespace=~"tidb-management-service",container!="",image!=""}[5m])
              )
              /
              clamp_min(
                sum by (namespace,pod) (
                  kube_pod_container_resource_limits{namespace=~"tidb-management-service",resource="cpu"}
                ),
                0.001
              )
            ) > 90
          )
          and on (namespace,pod)
          (
            max by (namespace,pod) (
              kube_pod_status_phase{namespace=~"tidb-management-service",phase="Running"} == 1
            )
          )
          and on (namespace,pod)
          (
            sum by (namespace,pod) (
              kube_pod_container_resource_limits{namespace=~"tidb-management-service",resource="cpu"}
            ) > 0
          )
        for: 10m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: MemoryHigh
        annotations:
          message: Memory > 85% ns={{ $labels.namespace }} pod={{ $labels.pod }}
        expr: |
          (
            100 *
            (
              sum by (namespace,pod) (
                container_memory_working_set_bytes{namespace=~"tidb-management-service",container!="",image!=""}
              )
              /
              clamp_min(
                sum by (namespace,pod) (
                  kube_pod_container_resource_limits{namespace=~"tidb-management-service",resource="memory"}
                ),
                1
              )
            ) > 85
          )
          and on (namespace,pod)
          (
            max by (namespace,pod) (
              kube_pod_status_phase{namespace=~"tidb-management-service",phase="Running"} == 1
            )
          )
          and on (namespace,pod)
          (
            sum by (namespace,pod) (
              kube_pod_container_resource_limits{namespace=~"tidb-management-service",resource="memory"}
            ) > 0
          )
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: MemoryVeryHigh
        annotations:
          message: Memory > 90% ns={{ $labels.namespace }} pod={{ $labels.pod }}
        expr: |
          (
            100 *
            (
              sum by (namespace,pod) (
                container_memory_working_set_bytes{namespace=~"tidb-management-service",container!="",image!=""}
              )
              /
              clamp_min(
                sum by (namespace,pod) (
                  kube_pod_container_resource_limits{namespace=~"tidb-management-service",resource="memory"}
                ),
                1
              )
            ) > 90
          )
          and on (namespace,pod)
          (
            max by (namespace,pod) (
              kube_pod_status_phase{namespace=~"tidb-management-service",phase="Running"} == 1
            )
          )
          and on (namespace,pod)
          (
            sum by (namespace,pod) (
              kube_pod_container_resource_limits{namespace=~"tidb-management-service",resource="memory"}
            ) > 0
          )
        for: 10m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: PodRestarts10m
        annotations:
          message: Restarts in 10m {{ printf "%.0f" $value }} ns={{ $labels.namespace }} pod={{ $labels.pod }}
        expr: |
          sum by (namespace,pod) (
            increase(kube_pod_container_status_restarts_total{namespace=~"tidb-management-service"}[10m])
          ) > 5
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: PodRestarts30m
        annotations:
          message: Restarts in 30m {{ printf "%.0f" $value }} ns={{ $labels.namespace }} pod={{ $labels.pod }}
        expr: |
          sum by (namespace,pod) (
            increase(kube_pod_container_status_restarts_total{namespace=~"tidb-management-service"}[30m])
          ) > 10
        for: 10m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: PodNotReady
        annotations:
          message: Not Ready ns={{ $labels.namespace }} pod={{ $labels.pod }}
        expr: |
          (
            max by (namespace,pod) (
              kube_pod_status_ready{namespace=~"tidb-management-service",condition="true"} == 0
            )
          )
          and on (namespace,pod)
          (
            max by (namespace,pod) (
              kube_pod_status_phase{namespace=~"tidb-management-service",phase="Running"} == 1
            )
          )
          unless on (namespace,pod)
          (
            kube_pod_deletion_timestamp{namespace=~"tidb-management-service"} > 0
          )
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: PodEndpointDown
        annotations:
          message: Endpoint down while pod is Ready ns={{ $labels.namespace }} pod={{ $labels.pod }}
        expr: |
          (
            max_over_time(up{namespace=~"tidb-management-service"}[5m]) == 0
          )
          and on (namespace,pod)
          (
            max by (namespace,pod) (
              kube_pod_status_phase{namespace=~"tidb-management-service",phase="Running"} == 1
            )
          )
          and on (namespace,pod)
          (
            max by (namespace,pod) (
              kube_pod_status_ready{namespace=~"tidb-management-service",condition="true"} == 1
            )
          )
          unless on (namespace,pod)
          (
            kube_pod_deletion_timestamp{namespace=~"tidb-management-service"} > 0
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: DeploymentReplicasUnavailable
        annotations:
          message: Deployment has unavailable replicas ns={{ $labels.namespace }} deployment={{ $labels.deployment }}
        expr: |
          kube_deployment_status_replicas_unavailable{namespace=~"tidb-management-service"} > 0
        for: 10m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: CrashLoopBackOff
        annotations:
          message: CrashLoopBackOff ns={{ $labels.namespace }} pod={{ $labels.pod }} container={{ $labels.container }}
        expr: |
          max by (namespace,pod,container) (
            kube_pod_container_status_waiting_reason{
              namespace=~"tidb-management-service",
              reason="CrashLoopBackOff"
            } == 1
          )
        for: 10m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: OOMKilled
        annotations:
          message: OOMKilled ns={{ $labels.namespace }} pod={{ $labels.pod }} container={{ $labels.container }}
        expr: |
          sum by (namespace,pod,container) (
            increase(kube_pod_container_status_restarts_total{namespace=~"tidb-management-service"}[5m])
          ) > 0
          and on (namespace,pod,container)
          kube_pod_container_status_last_terminated_reason{
            namespace=~"tidb-management-service",
            reason="OOMKilled"
          } == 1
        for: 1m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: OOMKilled30m
        annotations:
          message: Frequent OOMKilled in 30m ns={{ $labels.namespace }} pod={{ $labels.pod }} container={{ $labels.container }}
        expr: |
          sum by (namespace,pod,container) (
            increase(kube_pod_container_status_restarts_total{namespace=~"tidb-management-service"}[30m])
          ) > 2
          and on (namespace,pod,container)
          kube_pod_container_status_last_terminated_reason{
            namespace=~"tidb-management-service",
            reason="OOMKilled"
          } == 1
        for: 2m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: ImagePullBackOff
        annotations:
          message: ImagePullBackOff ns={{ $labels.namespace }} pod={{ $labels.pod }} container={{ $labels.container }}
        expr: |
          max by (namespace,pod,container) (
            kube_pod_container_status_waiting_reason{
              namespace=~"tidb-management-service",
              reason="ImagePullBackOff"
            } == 1
          )
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: PodPendingLong
        annotations:
          message: Pending > 15m ns={{ $labels.namespace }} pod={{ $labels.pod }}
        expr: |
          max by (namespace,pod) (
            kube_pod_status_phase{
              namespace=~"tidb-management-service",
              phase="Pending"
            } == 1
          )
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control

      - alert: PodEvicted
        annotations:
          message: Evicted ns={{ $labels.namespace }} pod={{ $labels.pod }}
        expr: |
          max by (namespace,pod) (
            kube_pod_status_reason{
              namespace=~"tidb-management-service",
              reason="Evicted"
            } == 1
          )
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control