apiVersion: "monitoring.coreos.com/v1"
kind: "PrometheusRule"
metadata:
  name: "self-monitoring-serverless-region-svc-alert-rules"
  namespace: "tidb-management-service"
  labels:
    relase: "self-monitoring"
spec:
  groups:
    - name: serverless-regional-service
      rules:
        - alert: CPUUsageIsTooHigh
          annotations:
            message: CPU utilization of pod {{ $labels.pod }} is more than 90%.
          expr: |
            (sum(node_namespace_pod_container:container_cpu_usage_seconds_total:sum_rate{namespace=~"tidb-management-service"}) by (namespace,pod)/
              sum(kube_pod_container_resource_limits{resource="cpu",namespace=~"tidb-management-service"}) by (namespace,pod)
            ) *100 > 80
          for: 3m
          labels:
            severity: critical
            component: serverless-svc

        - alert: MemUsageIsTooHigh
          annotations:
            message: Memory utilization of pod {{ $labels.pod }} is more than 90%.
          expr: |
            (sum(container_memory_working_set_bytes{namespace=~"tidb-management-service",container!="POD", container!="", image!=""})by(pod)/
            sum(kube_pod_container_resource_limits{resource="memory",namespace=~"tidb-management-service"}) by (pod)
            * 100) > 70
          for: 3m
          labels:
            severity: critical
            component: serverless-svc

        - alert: PodRestarts
          annotations:
            summary: The pod {{ $labels.pod }} restart frequently
            message: Container {{ $labels.namespace }}/{{ $labels.container }} is restarting {{ printf "%.2f" $value }} times.
          expr: |
            increase(kube_pod_container_status_restarts_total{namespace="tidb-management-service"}[5m]) > 0
          labels:
            severity: warning
            component: serverless-svc

        - alert: PodRestartsFrequently
          annotations:
            summary: The pod {{ $labels.pod }} restart frequently
            message: Container {{ $labels.namespace }}/{{ $labels.container }} is restarting {{ printf "%.2f" $value }} times.
          expr: |
            increase(kube_pod_container_status_restarts_total{namespace="tidb-management-service"}[5m]) > 3
          labels:
            severity: critical
            component: serverless-svc

        - alert: PodIsDown
          annotations:
            summary: The pod {{ $labels.pod }} is down
            message: The pod {{ $labels.namespace }} / {{ $labels.pod }} is down.
          expr: |
            up{namespace=~"tidb-management-service"} == 0
          for: 3m
          labels:
            severity: critical
            component: serverless-svc