# apiVersion: monitoring.coreos.com/v1
# kind: PrometheusRule
# metadata:
#   labels:
#     release: self-monitoring
#   name: metrics-integration-k8s-alert
#   namespace: self-monitoring
# spec:
#   groups:
#     - name: metrics-integration.rules
#       rules:
#         - alert: KubePodCrashLooping
#           annotations:
#             message: Pod {{ $labels.namespace }}/{{ $labels.pod }} is restarting {{ printf "%.2f" $value }} times in 1 hours.
#             runbook_url: https://github.com/tidbcloud/runbooks/wiki/KubePodCrashLooping
#           expr: |
#             increase(kube_pod_container_status_restarts_total{namespace=~"pipeline-.*"}[1h]) > 5
#           for: 1m
#           labels:
#             component: metrics-integration
#             severity: critical
#         - alert: KubePodNotReady
#           annotations:
#             message: Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in a non-ready state for longer than 15 minutes.
#             runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodnotready
#           expr: |
#             sum by (namespace, pod) (max by(namespace, pod) (kube_pod_status_phase{phase=~"Pending|Unknown", namespace=~"pipeline-.*"}) * on(namespace, pod) group_left(owner_kind) max by(namespace, pod, owner_kind) (kube_pod_owner{owner_kind!="Job"})) > 0
#           for: 15m
#           labels:
#             severity: warning
#             component: metrics-integration
#         - alert: UnhealthyPod
#           annotations:
#             summary: The pod {{ $labels.pod }} is not ready
#             message: Pod {{ $labels.namespace }}/{{ $labels.pod }} in {{ $labels.cluster }} has been in a non-ready state for longer than 2 minutes.
#           expr: |
#             kube_pod_status_phase{phase=~"Pending|Unknown|Failed",namespace=~"pipeline-.*"} == 1
#           for: 2m
#           labels:
#             severity: critical
#             component: metrics-integration