groups:
  - name: nextgen.br
    rules:
      # backup pod alerts
      - alert: NextGenBackupPodNotNormalTooLong
        annotations:
          message: Pod {{ $labels.namespace }}/{{ $labels.pod }} is {{ $labels.phase }} for more than 60 minutes.
        expr: |
          max_over_time(kube_pod_status_phase{namespace=~"tidb.*", pod=~"^(tibr-default).*", phase!~"Running|Succeeded"}[5m]) == 1
        for: 60m
        labels:
          severity: critical
          response_class: page
          tier: next-gen
          component: next-gen-br-kernel
          oncall_team: tidb_x_tools
      - alert: NextGenBackupPodCrashLooping
        annotations:
          message:
            Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) is restarting {{ printf "%.2f" $value }} times in 1 hours.
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace=~"tidb.*", pod=~"^(tibr-default).*"}[1h]) > 5
        for: 10m
        labels:
          severity: critical
          response_class: page
          tier: next-gen
          component: next-gen-br-kernel
          oncall_team: tidb_x_tools
      - alert: NextGenBackupPodRestart
        annotations:
          message:
            Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) is restarting {{ printf "%.2f" $value }} times in 1 hours.
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace=~"tidb.*", pod=~"^(tibr-default).*"}[10m]) > 1
        for: 10m
        labels:
          severity: major
          response_class: ticket
          tier: next-gen
          component: next-gen-br-kernel
          oncall_team: tidb_x_tools
      # restore pod alerts
      - alert: NextGenRestorePodNotNormalTooLong
        annotations:
          message:
            Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) is {{ $labels.phase }} for more than 20 minutes.
        expr: |
          max_over_time(kube_pod_status_phase{namespace=~"tidb.*", pod=~"^(restore).*", phase!~"Running|Succeeded"}[5m]) == 1
        for: 20m
        labels:
          severity: critical
          response_class: page
          tier: next-gen
          component: next-gen-br-kernel
          oncall_team: tidb_x_tools
      - alert: NextGenRestorePodRestart
        annotations:
          message:
            Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) is restarting {{ printf "%.2f" $value }} times in 1 hours.
        expr: |
          increase(kube_pod_container_status_restarts_total{namespace=~"tidb.*", pod=~"^(restore).*"}[10m]) > 1
        for: 10m
        labels:
          severity: critical # pod restart will cause restore failed
          response_class: page
          tier: next-gen
          component: next-gen-br-kernel
          oncall_team: tidb_x_tools
      # tibrgc pod alerts
      - alert: NextGenTibrgcPodNotNormalTooLong
        annotations:
          message:
            Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) is {{ $labels.phase }} for more than 20 minutes.
        expr: |
          max_over_time(kube_pod_status_phase{namespace=~"tidb.*", pod=~"^(tibrgc).*", phase!~"Running|Succeeded"}[5m]) == 1
        for: 20m
        labels:
          severity: major
          response_class: ticket
          tier: next-gen
          component: next-gen-br-kernel
          oncall_team: tidb_x_tools
      - alert: NextGenTibrgcPodCrashLooping
        annotations:
          message:
            Pod {{ $labels.namespace }}/{{ $labels.pod }} ({{ $labels.container
            }}) is restarting {{ printf "%.2f" $value }} times in 1 hours.
        expr: |
          increase(kube_pod_container_status_restarts_total{ namespace=~"tidb.*", pod=~"^(tibrgc).*"}[1h]) > 5
        for: 10m
        labels:
          severity: major
          response_class: ticket
          tier: next-gen
          component: next-gen-br-kernel
          oncall_team: tidb_x_tools