groups:
- name: self-monitoring-o11y-base
  rules:
  - alert: O11yPodMemoryFillingUp
    annotations:
      description: The pod name {{ $labels.pod
        }} in Namespace {{ $labels.namespace }} memory usage is above 85% limit
      summary: pod memory limit is filling up.
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcniKUi42cYGwQESW3nBkcrqb
    expr: |-
      (
              sum by (pod, namespace) (
                container_memory_working_set_bytes{image!="",namespace!~"dataservice|tidb-management-service", pod=~".*vmalert.*"}
              )
            /
              sum by (pod, namespace) (
                kube_pod_container_resource_limits{namespace!~"dataservice|tidb-management-service", pod=~".*vmalert.*", resource="memory"}
              )
          *
            100
        !=
          +Inf
      ) > 85
    for: 5m
    labels:
      oncall_team: o11y
      severity: warning
  - alert: O11yHostNodeCPUHigh
    annotations:
      description: The node name {{ $labels.instance }}  cpu usage > 85%
      summary: o11y host CPU high.
    expr: |-
      100 * (1 - avg(irate(node_cpu_seconds_total{mode="idle"}[2m])) by(instance)) > 85
    for: 15m
    labels:
      oncall_team: o11y
      severity: warning
  - alert: O11yHostNodeMemoryHigh
    annotations:
      description: The node name {{ $labels.instance }}  memory usage > 85%
      summary: o11y host Memory high.
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnkBuJwKshPn0b0O66SEGRec
    expr: |-
      (node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) / node_memory_MemTotal_bytes * 100 > 85
    for: 30m
    labels:
      oncall_team: o11y
      severity: warning
  - alert: PrometheusCPUThrottlingHigh
    annotations:
      description: throttling of CPU in namespace {{ $labels.namespace }} for  pod {{ $labels.pod }} .
      summary: Processes experience elevated CPU throttling.
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcniKUi42cYGwQESW3nBkcrqb
    expr: |-
      sum(increase(container_cpu_cfs_throttled_periods_total{container="prometheus"}[5m])) by (container, pod, namespace)
        /
      sum(increase(container_cpu_cfs_periods_total{}[5m])) by (container, pod, namespace)
        > ( 75 / 100 )
    for: 15m
    labels:
      oncall_team: o11y
      severity: warning
  - alert: PodOOMKilled
    expr: |-
      increase(kube_pod_container_status_restarts_total{pod=~".*vmalert.*"}[5m]) >= 1
      and ignoring (reason)
      kube_pod_container_status_last_terminated_reason{pod=~".*vmalert.*",reason="OOMKilled"} == 1
    for: 3m
    labels:
      oncall_team: o11y
      severity: warning
    annotations:
      description: "VMAlert Pod in {{ $labels.namespace }} has been OOMKilled {{ $value }} times in the last 5 minutes.\n LABELS = {{ $labels }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnVA3bxgyU37DWJxLRyQlHRg
  - alert: PodOOMKilled
    expr: |-
      increase(kube_pod_container_status_restarts_total{namespace!~"dataservice|tidb-management-service|drhouse|observability"}[10m]) >= 2
      and ignoring (reason)
      kube_pod_container_status_last_terminated_reason{namespace!~"dataservice|tidb-management-service|drhouse|observability",reason="OOMKilled"} == 1
    for: 3m
    labels:
      oncall_team: o11y
      severity: major
    annotations:
      description: "Pod in {{ $labels.namespace }} has been OOMKilled {{ $value }} times in the last 10 minutes.\n LABELS = {{ $labels }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnVA3bxgyU37DWJxLRyQlHRg
  - alert: O11yPodOOMKilled
    expr: |-
      increase(kube_pod_container_status_restarts_total{namespace=~"drhouse|observability", container!="vmbackuper"}[5m]) >= 1
      and ignoring (reason)
      kube_pod_container_status_last_terminated_reason{namespace=~"drhouse|observability", container!="vmbackuper", reason="OOMKilled"} == 1
    for: 3m
    labels:
      oncall_team: o11y
      severity: major
    annotations:
      description: "Pod in {{ $labels.namespace }} has been OOMKilled {{ $value }} times in the last 5 minutes.\n LABELS = {{ $labels }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnVA3bxgyU37DWJxLRyQlHRg
  - alert: O11yPodOOMKilledFrequently
    expr: |-
      increase(kube_pod_container_status_restarts_total{namespace="observability"}[15m]) >= 3
      and ignoring (reason)
      kube_pod_container_status_last_terminated_reason{namespace="observability",reason="OOMKilled"} == 1
    for: 3m
    labels:
      oncall_team: o11y
      severity: critical
    annotations:
      description: "Pod in {{ $labels.namespace }} has been OOMKilled {{ $value }} times in the last 15 minutes.\n LABELS = {{ $labels }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnVA3bxgyU37DWJxLRyQlHRg
  - alert: O11yNginxPodOOMKilled
    expr: |-
      increase(kube_pod_container_status_restarts_total{namespace=~"kube-system|ingress-nginx-azall", pod=~".*nginx.*"}[5m]) >= 1
      and ignoring (reason)
      kube_pod_container_status_last_terminated_reason{namespace=~"kube-system|ingress-nginx-azall", reason="OOMKilled", pod=~".*nginx.*"} == 1
    for: 5m
    labels:
      oncall_team: o11y
      severity: critical
    annotations:
      description: "Pod in {{ $labels.namespace }} has been OOMKilled {{ $value }} times in the last 5 minutes.\n LABELS = {{ $labels }}"
      runbook_url: https://pingcap.feishu.cn/wiki/wikcnJ3TYvhozrRtWCMBh5z92Wd#doxcnVA3bxgyU37DWJxLRyQlHRg
  - alert: CoreDNSLatencyHigh
    annotations:
      description: CoreDNS in {{ $labels.cluster }} has 99th percentile latency of {{ $value }} seconds.
      summary: CoreDNS is experiencing high 99th percentile latency.
    expr: |
      histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{job="coredns"}[5m])) without (instance,pod)) > 2
    for: 10m
    labels:
      oncall_team: o11y
      severity: major
  - alert: CoreDNSForwardLatencyHigh
    annotations:
      description: CoreDNS in {{ $labels.cluster }} has 99th percentile latency of {{ $value }} seconds forwarding  requests.
      summary: CoreDNS is experiencing high latency forwarding requests.
    expr: |
      histogram_quantile(0.99, sum(rate(coredns_forward_request_duration_seconds_bucket{job="coredns"}[5m])) without (pod, instance, rcode)) > 4
    for: 10m
    labels:
        oncall_team: o11y
        severity: major
  - alert: ENA_Bandwidth_In_Allowance_Exceeded
    expr: |
      rate(node_ethtool_bw_in_allowance_exceeded{}[30s]) > 100
    for: 1h
    labels:
      oncall_team: o11y
      severity: major
    annotations:
      message: "Device {{ $labels.device }} of Instance {{ $labels.instance }} in {{ $labels.cluster }} bandwidth-in exceeded"
      description: "Instance {{ $labels.instance }} from {{ $labels.cluster }} has a bandwidth in allowance greater than {{ $value }}"
  - alert: ENA_Bandwidth_Out_Allowance_Exceeded
    expr: |
      rate(node_ethtool_bw_out_allowance_exceeded{}[30s]) > 100
    for: 1h
    labels:
      oncall_team: o11y
      severity: major
    annotations:
      message: "Device {{ $labels.device }} of Instance {{ $labels.instance }} in {{ $labels.cluster }} bandwith-out traffic exceeded"
      description: "Instance {{ $labels.instance }} from {{ $labels.cluster }} has a bandwidth out allowance greater than {{ $value }}"
  - alert: ENA_PPS_Allowance_Exceeded
    expr: |
      rate(node_ethtool_pps_allowance_exceeded{}[30s]) > 0
    for: 30m
    labels:
      oncall_team: o11y
      severity: major
    annotations:
      message: "Device {{ $labels.device }} of Instance {{ $labels.instance }} in {{ $labels.cluster }} (PPS) allowance exceeded"
      description: "Packet Per Second (PPS) allowance for instance {{ $labels.instance }} in {{ $labels.cluster }} is greater than 0. pps_allowance_exceeded indicates number of packets queued or dropped due to Packet Per Second (PPS) exceeding the allowance for the instance. PPS allowance is enforced separately to the overall bandwidth allowance and, while the instance may still be under overall bandwidth allowance, the PPS allowance may exceed if the mean packet size is small."
  - alert: ENA_Conntrack_Allowance_Exceeded
    expr: |
      rate(node_ethtool_conntrack_allowance_exceeded{}[30s]) > 0
    for: 30m
    labels:
      oncall_team: o11y
      severity: major
    annotations:
      message: "Device {{ $labels.device }} of Instance {{ $labels.instance }} in {{ $labels.cluster }} conntrack allowance exceeded"
      description: "conntrack_allowance_exceeded indicates the number of packets dropped due to exhaustion of tracked session allowance for the instance, new sessions will fail to establish once this allowance is exceeded."

  - alert: ENA_Linklocal_Allowance_Exceeded
    expr: |
      rate(node_ethtool_linklocal_allowance_exceeded{}[30s]) > 0
    for: 30m
    labels:
      oncall_team: o11y
      severity: major
    annotations:
      message: "Device {{ $labels.device }} of Instance {{ $labels.instance }} in {{ $labels.cluster }} linklocal allowance exceeded"
      description: "linklocal_allowance_exceeded indicates the number of packets dropped due to PPS rate allowance exceeded for local services such as Route 53 DNS Resolver, Instance Metadata Service, Amazon Time Sync Service. This often points to suboptimal design choices or misconfiguration. This allowance is the same across instances."

  - alert: MetricsStatusCollectorFailed
    expr: |
      (kube_job_status_failed{job_name=~"metrics-status-collector-.*"} unless ((kube_job_status_failed{job_name=~"metrics-status-collector-.*"} offset 5m) > 0)) > 0
    for: 2m
    labels:
      oncall_team: o11y
      severity: major
    annotations:
      summary: "Metrics status collector job has failed."
      message: "Metrics status collector in {{ $labels.namespace }} has failed. Please check the logs for more details and retry the job after fixing the issue."