apiVersion: monitoring.coreos.com/v1
kind: PrometheusRule
metadata:
  labels:
    release: self-monitoring
  name: metrics-integration-k8s-alert
  namespace: self-monitoring
spec:
  groups:
    - name: metrics-integration.rules
      rules:
        - alert: KubePodCrashLooping
          annotations:
            message: Pod {{ $labels.namespace }}/{{ $labels.pod }} is restarting {{ printf "%.2f" $value }} times in 1 hours.
            runbook_url: https://github.com/tidbcloud/runbooks/wiki/KubePodCrashLooping
          expr: |
            increase(kube_pod_container_status_restarts_total{namespace=~"pipeline-.*"}[1h]) > 5
          for: 1m
          labels:
            component: metrics-integration
            severity: critical
        - alert: KubePodNotReady
          annotations:
            message: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.
            runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready
          expr: |
            sum by (namespace, pod) (max by(namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown", namespace=~"pipeline-.*"}) * on(namespace, pod) group_left(owner_kind) max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})) > 0
          for: 15m
          labels:
            severity: warning
            component: metrics-integration
        - alert: UnhealthyPod
          annotations:
            summary: The pod {{ $labels.pod }} is not ready
            message: Pod {{ $labels.namespace }}/{{ $labels.pod }} in {{ $labels.cluster }} has been in a non-ready state for longer than 2 minutes.
          expr: |
            kube_pod_status_phase{phase=~"Pending|Unknown|Failed",namespace=~"pipeline-.*"} == 1
          for: 2m
          labels:
            severity: critical
            component: metrics-integration