groups:
  - name: csng.dataplane
    rules:
      - alert: NextGenPodRestartsFrequently
        annotations:
          summary: The pod {{ $labels.pod }} in Namespace {{ $labels.namespace }} on sharedpool {{ $labels.tidb_cluster_id }} restart frequently
          message: Container {{ $labels.namespace }}/{{ $labels.container }} is restarting {{ printf "%.2f" $value }} times.
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace=~"tidb.*", pod!~".*(tibr|tibrgc|restore|o11y|cdc).*"}[2m]) > 2
        for: 5m
        labels:
          severity: critical
          response_class: page
          oncall_team: premium
          tier: next-gen

      - alert: NextGenPodCrashLooping
        annotations:
          message: |
            Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) on sharedpool {{ $labels.tidb_cluster_id }} is restarting {{ printf "%.2f" $value }} times in 1 hours.
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace=~"tidb.*", pod!~".*(tibr|tibrgc|restore|o11y|cdc|tikv|pd|tidb|tiflash).*"}[1h]) > 5
        for: 10m
        labels:
          oncall_team: premium
          severity: critical
          response_class: page
          tier: next-gen
      - alert: NextGenTiKVPodCrashLooping
        annotations:
          message: |
            Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) on sharedpool {{ $labels.tidb_cluster_id }} is restarting {{ printf "%.2f" $value }} times in 1 hours.
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace=~"tidb.*", pod=~".*tikv.*"}[1h]) > 5
        for: 10m
        labels:
          component: tikv
          oncall_team: tikv_unified
          severity: critical
          response_class: page
          tier: next-gen
      - alert: NextGenPDPodCrashLooping
        annotations:
          message: |
            Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) on sharedpool {{ $labels.tidb_cluster_id }} is restarting {{ printf "%.2f" $value }} times in 1 hours.
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace=~"tidb.*", pod=~".*pd.*"}[1h]) > 5
        for: 10m
        labels:
          component: pd
          oncall_team: pd
          severity: critical
          response_class: page
          tier: next-gen
      - alert: NextGenTidbPodCrashLooping
        annotations:
          message: |
            Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) on sharedpool {{ $labels.tidb_cluster_id }} is restarting {{ printf "%.2f" $value }} times in 1 hours.
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace=~"tidb.*", pod=~".*tidb.*"}[1h]) > 5
        for: 10m
        labels:
          oncall_team: tidb_executor
          severity: critical
          response_class: page
          tier: next-gen
      - alert: NextGenTiflashPodCrashLooping
        annotations:
          message: |
            Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) on sharedpool {{ $labels.tidb_cluster_id }} is restarting {{ printf "%.2f" $value }} times in 1 hours.
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace=~"tidb.*", pod=~".*tiflash.*"}[1h]) > 5
        for: 10m
        labels:
          component: tiflash
          oncall_team: tiflash
          severity: critical
          response_class: page
          tier: next-gen

      - alert: NextGenPodIsNotRunning
        expr: kube_pod_status_phase{phase!="Running", phase!="Succeeded", namespace=~"tidb.*", pod!~".*(tibr|tibrgc|restore|o11y|cdc).*"} == 1
        for: 10m
        labels:
          severity: critical
          response_class: page
          tier: next-gen
          component: cluster-ng-service
          oncall_team: premium
        annotations:
          runbook_url: https://docs.google.com/document/d/17dQQS3ONxhqdlqKth9kZHTtpANp49UgUBF_dMk5ewCE/edit#heading=h.f5wwrkav7k5f
          value: "{{ $value }}"
          message: "Pod {{ $labels.namespace }}/{{ $labels.pod }} has been in {{ $labels.phase }} status for more than 10 minutes."

      - alert: NextGenNodeFilesystemAlmostOutOfSpace
        annotations:
          message: |
            Filesystem on {{ $labels.device }} at {{ $labels.instance }} on sharedpool {{ $labels.tidb_cluster_id }}
            has only {{ printf "%.2f" $value }}% available space left.
          runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-nodefilesystemalmostoutofspace
          summary: Filesystem has less than 20% space left.
        expr: |
          (
            node_filesystem_avail_bytes{job="node-exporter",fstype!~"erofs|"} / node_filesystem_size_bytes{job="node-exporter",fstype!~"erofs|"} * 100 < 20
          and
            node_filesystem_readonly{job="node-exporter",fstype!~"erofs|"} == 0
          )
        for: 10m
        labels:
          component: cluster-ng-service
          oncall_team: premium
          severity: critical
          response_class: page
          tier: next-gen

      - alert: NextGenCronJobFailed
        annotations:
          message: "CronJob job {{ $labels.namespace }}/{{ $labels.job_name }} on sharedpool {{ $labels.tidb_cluster_id }} failed within the last hour."
          runbook_url: https://github.com/tidbcloud/runbooks/wiki/CronJobFailed
          summary: "CronJob job {{ $labels.job_name }} failed recently"
        expr: |
          kube_job_status_failed{namespace=~"tidb.*", job_name!~"tibrgc.*"} > 0
          and on(namespace, job_name)
          kube_job_status_succeeded{namespace=~"tidb.*"} == 0
          and on(namespace, job_name)
          kube_job_owner{owner_kind="CronJob", namespace=~"tidb.*"}
          and on(namespace, job_name)
          time() - kube_job_created{namespace=~"tidb.*"} < 3600
        for: 10m
        labels:
          severity: critical
          response_class: page
          component: cluster-ng-service
          oncall_team: premium
          tier: next-gen