groups:
  - name: csng.br.runtime
    rules:
      - alert: NextGenRegionalFlushWALFailure
        annotations:
          message: Flush WAL failed and the workflow continued. shared_pool_id={{ $labels.shared_pool_id }}
        expr: increase(flush_wal_failure_total[5m]) > 0
        labels:
          severity: major
          response_class: ticket
          component: next-gen-br-kernel
          oncall_team: br
          tier: next-gen

      - alert: ExecutorBRTaskTooManyErrors
        annotations:
          message: Runtime Executor BR task {{ $labels.task_name }} action {{ $labels.action_name }} has too many errors on region {{ $labels.tidb_cluster_id }}.
        expr: |
          sum by (executor_name, action_name, task_name, tidb_cluster_id) (increase(runtime_executor_dag_action_duration_seconds_count{result="error", task_name=~"mb(-del)?-.*|.*(backup|restore).*", action_name!~"clean-up-cert|pre-create-tidbcluster|pre-create-tidbgroup|apply-pd-group"}[5m])) >= 20
        for: 1m
        labels:
          severity: critical
          response_class: page
          component: next-gen-br
          oncall_team: tidb_x_tools
          tier: next-gen

      - alert: ExecutorBRTaskRunningTooLong
        annotations:
          message: Runtime Executor BR task {{ $labels.task_name }} action {{ $labels.action_name }} is running too long on region {{ $labels.tidb_cluster_id }}.
        expr: |
          max by(executor_name, task_name, action_name, tidb_cluster_id) (
            runtime_executor_dag_action_running_walltime_seconds{task_name=~"mb(-del)?-.*|.*(backup|restore).*", action_name!="clear-dataplane", result="running"}
          ) > 14400
        for: 1m
        labels:
          severity: critical
          response_class: page
          component: next-gen-br
          oncall_team: tidb_x_tools
          tier: next-gen