groups:
  - name: dedicated-regionalserver-pod
    rules:
      - alert: CPUUsageIsTooHigh
        annotations:
          message: CPU utilization of pod {{ $labels.pod }} is more than 80%.
          diagnosis: 'click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left={"datasource":"Ak6VpZD4z","queries":[{"refId":"A","editorMode":"code","expr":"(sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{ pod=\"{{ $labels.pod }}\"}) by (pod)/sum(kube_pod_container_resource_limits{resource=\"cpu\", pod=\"{{ $labels.pod }}\"}) by (pod)) *100","legendFormat":"__auto","range":true,"instant":true,"hide":false}],"range":{"from":"now-1h","to":"now"}}'
        expr: |
          (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=~"regional-server-system", pod=~"regional-server.*"}) by (namespace,pod)/
            sum(kube_pod_container_resource_limits{resource="cpu",namespace=~"regional-server-system", pod=~"regional-server.*"}) by (namespace,pod)
          ) *100 > 80
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: dedicated-regional-server
          oncall_team: dedicated_service

      - alert: MemUsageIsTooHigh
        annotations:
          message: Memory utilization of pod {{ $labels.pod }} is more than 80%.
          diagnosis: 'click here to see detail: https://clinic.pingcap.com/grafana/explore?orgId=1&left={"datasource":"Ak6VpZD4z","queries":[{"refId":"A","editorMode":"code","expr":"(sum(container_memory_working_set_bytes{container!=\"POD\", container!=\"\", image!=\"\", pod=\"{{ $labels.pod }}\"})by(pod)/sum(kube_pod_container_resource_limits{resource=\"memory\", pod=\"{{ $labels.pod }}\"}) by (pod) * 100)","legendFormat":"__auto","range":true,"instant":true,"hide":false}],"range":{"from":"now-1h","to":"now"}}'
        expr: |
          (sum(container_memory_working_set_bytes{namespace=~"regional-server-system", pod=~"regional-server.*",container!="POD", container!="", image!="",container!="serverless-global-regional-proxy"})by(pod)/
          sum(kube_pod_container_resource_limits{resource="memory",namespace=~"regional-server-system", pod=~"regional-server.*"}) by (pod)
          * 100) > 80
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: dedicated-regional-server
          oncall_team: dedicated_service

      - alert: PodRestartsFrequently
        annotations:
          summary: The pod {{ $labels.pod }} restart frequently
          message: Container {{ $labels.namespace }}/{{ $labels.container }} is restarting {{ printf "%.2f" $value }} times.
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace=~"regional-server-system", pod=~"regional-server.*"}[2m]) > 2
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: dedicated-regional-server
          oncall_team: dedicated_service

      - alert: PodIsDown
        annotations:
          summary: The pod {{ $labels.pod }} is down
          message: The pod {{ $labels.namespace }} / {{ $labels.pod }} is down.
        expr: |
          up{namespace=~"regional-server-system", pod=~"regional-server.*"} == 0
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: dedicated-regional-server
          oncall_team: dedicated_service