groups:
- name: tikv.pod.rules
  rules:
  - alert: WorkerCPUUsageIsTooHigh
    annotations:
      # Unable to get control_plane_info. Hardcode to us-west-2-f01.
      message: CPU utilization of pod {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 80% for 5m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-pod={{ $labels.pod }}&var-namespace=tidb-serverless&from=now-30m&to=now)
    expr: |
      (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace="tidb-serverless",container=~"tikv-worker|cop-worker|replication-worker|tikv-api|schema-manager|rep-cdc|ticdc"}) by (pod) /
        sum(kube_pod_container_resource_limits{resource="cpu", namespace="tidb-serverless",container=~"tikv-worker|cop-worker|replication-worker|tikv-api|schema-manager|rep-cdc|ticdc"}) by (pod)
      ) * 100 > 80
    for: 5m
    labels:
      severity: warning
      component: tikv
      oncall_team: tikv_unified
  - alert: TiKVCPUUsageIsTooHigh
    annotations:
      # Unable to get control_plane_info. Hardcode to us-west-2-f01.
      message: CPU utilization of pod {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 80% for 5m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-pod={{ $labels.pod }}&var-namespace=tidb-serverless&from=now-30m&to=now)
    expr: |
      (sum(rate(process_cpu_seconds_total{namespace="tidb-serverless",container="tikv"}[1m])) by (pod) /
        on(namespace, pod)
        sum(tikv_server_cpu_cores_quota{namespace="tidb-serverless",container="tikv"}) by (pod)
      ) * 100 > 80
    for: 5m
    labels:
      severity: warning
      component: tikv
      oncall_team: tikv_unified
  - alert: TiKVCPUUsageIsTooHigh
    annotations:
      # Unable to get control_plane_info. Hardcode to us-west-2-f01.
      message: CPU utilization of pod {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 90% for 15m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-pod={{ $labels.pod }}&var-namespace=tidb-serverless&from=now-30m&to=now)
    expr: |
      (sum(rate(process_cpu_seconds_total{namespace="tidb-serverless",container="tikv"}[1m])) by (namespace, pod) /
        on(namespace, pod)
        sum(tikv_server_cpu_cores_quota{namespace="tidb-serverless",container="tikv"}) by (namespace, pod)
      ) * 100 > 90
    for: 15m
    labels:
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified
  - alert: WorkerMemUsageIsTooHigh
    annotations:
      # Unable to get control_plane_info. Hardcode to us-west-2-f01.
      message: Memory utilization of pod {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 90% for 2m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-pod={{ $labels.pod }}&var-namespace=tidb-serverless&from=now-30m&to=now)
    expr: |
      (sum(container_memory_rss{namespace="tidb-serverless",container=~"tikv-worker|cop-worker|replication-worker|tikv-api|schema-manager|rep-cdc|ticdc"}) by (pod) /
        sum(kube_pod_container_resource_limits{resource="memory", namespace="tidb-serverless",container=~"tikv-worker|cop-worker|replication-worker|tikv-api|schema-manager|rep-cdc|ticdc"}) by (pod)
      ) * 100 > 90
    for: 2m
    labels:
      severity: warning
      component: tikv
      oncall_team: tikv_unified
  - alert: TiKVMemUsageIsTooHigh
    annotations:
      # Unable to get control_plane_info. Hardcode to us-west-2-f01.
      message: Memory utilization of pod {{ $labels.pod }} is {{ printf "%.2f" $value }}%, more than 90% for 2m. [Diagnosis Board](https://clinic.pingcap.com/grafana/d/zn-DjhE4z/serverless-pod-overview?orgId=1&refresh=30s&var-control_plane_info=us-west-2-f01%2Faws-us-west-2%2Fdevtier-infra&var-pod={{ $labels.pod }}&var-namespace=tidb-serverless&from=now-30m&to=now)
    expr: |
      (sum(container_memory_rss{namespace="tidb-serverless",container="tikv"}) by (pod) /
        sum(kube_pod_container_resource_limits{resource="memory", namespace="tidb-serverless",container="tikv"}) by (pod)
      ) * 100 > 90
    for: 2m
    labels:
      severity: critical
      response_class: page
      component: tikv
      oncall_team: tikv_unified