apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    release: self-monitoring
  name: self-monitoring-o11y-base
spec:
  groups:
  - name: self-monitoring-o11y-base
    rules:
    - alert: O11yPodMemoryFillingUp
      annotations:
        description: The pod name {{`{{ $labels.pod }}`}} in Namespace {{`{{ $labels.namespace }}`}} memory usage is above 85% limit
        summary: pod memory limit is filling up.
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcniKUi42cYGwQESW3nBkcrqb
      expr: |-
        (
                sum by (pod, namespace) (
                  container_memory_working_set_bytes{image!="",namespace!~"dataservice|tidb-management-service",pod!~"prometheus-self-monitoring-kube-prome-prometheus-0|.*operator.*|.*vmstorage.*"}
                )
              /
                sum by (pod, namespace) (
                  kube_pod_container_resource_limits{namespace!~"dataservice|tidb-management-service",resource="memory"}
                )
            *
              100
          !=
            +Inf
        ) > 85
      for: 5m
      labels:
        severity: warning
    - alert: O11yHostNodeCPUHigh
      annotations:
        description: The node name {{`{{ $labels.instance }}`}}  cpu usage > 85%
        summary: o11y host CPU high.
      expr: |-
        100 * (1 - avg(irate(node_cpu_seconds_total{mode="idle"}[2m])) by(instance)) > 85
      for: 15m
      labels:
        severity: warning
    - alert: O11yHostNodeMemoryHigh
      annotations:
        description: The node name {{`{{ $labels.instance }}`}}  memory usage > 85%
        summary: o11y host Memory high.
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnkBuJwKshPn0b0O66SEGRec
      expr: |-
        (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100 > 85
      for: 30m
      labels:
        severity: warning
    - alert: PrometheusCPUThrottlingHigh
      annotations:
        description: throttling of CPU in namespace {{`{{ $labels.namespace }}`}} for  pod {{`{{ $labels.pod }}`}} .
        summary: Processes experience elevated CPU throttling.
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcniKUi42cYGwQESW3nBkcrqb
      expr: |-
        sum(increase(container_cpu_cfs_throttled_periods_total{container="prometheus"}[5m])) by (container, pod, namespace)
          /
        sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)
          > ( 75 / 100 )
      for: 15m
      labels:
        severity: warning
    - alert: O11yPodOOMKilled
      expr: |-
        increase(kube_pod_container_status_restarts_total{namespace!~"dataservice|tidb-management-service"}[5m]) >= 1
        and ignoring (reason)
        kube_pod_container_status_last_terminated_reason{namespace!~"dataservice|tidb-management-service",reason="OOMKilled"} == 1
      for: 3m
      labels:
       severity: major
      annotations:
        description: "Pod in {{`{{ $labels.namespace }}`}} has been OOMKilled {{`{{ $value }}`}} times in the last 5 minutes.\n LABELS = {{`{{ $labels }}`}}"
        runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnVA3bxgyU37DWJxLRyQlHRg