groups:
  - name: csng.runtime
    rules:
      - alert: ExecutorAsyncTaskRunningTooLong
        annotations:
          message: Runtime Executor task {{ $labels.task_name }} action {{ $labels.action_name }} is running too long on region {{ $labels.tidb_cluster_id }}.
        expr: |
          max by(executor_name, task_name, action_name, tidb_cluster_id) (
            runtime_executor_dag_action_running_walltime_seconds{task_name=~"create-sharedpool-.*|create-tidbcluster-.*", result="running"}
          ) > 2700
        for: 1m
        labels:
          severity: critical
          response_class: page
          component: cluster-ng-service
          oncall_team: premium
          tier: next-gen

      - alert: ExecutorTaskRunningTooLong
        annotations:
          message: Runtime Executor task {{ $labels.task_name }} action {{ $labels.action_name }} is running too long on region {{ $labels.tidb_cluster_id }}.
        expr: |
          (
            (
              max by (executor_name, task_name, action_name, tidb_cluster_id) (
                runtime_executor_dag_action_running_walltime_seconds{
                  result="running",
                  task_name!~"create-sharedpool-.*|delete-sharedpool-.*|create-tidbcluster-.*|mb-.*|manual-backup-.*|delete-manual-backup-.*|create-lw-.*|delete-lw-.*|cl-restore-.*|del-cl-restore-.*|restore-.*"
                }
              )
            )
            or
            (
              max by (executor_name, task_name, action_name, tidb_cluster_id) (
                runtime_executor_dag_action_running_walltime_seconds{
                  result="running",
                  task_name=~"delete-sharedpool-.*",
                  action_name!="clear-dataplane"
                }
              )
            )
          ) > 1800
        for: 1m
        labels:
          severity: critical
          response_class: page
          component: cluster-ng-service
          oncall_team: premium
          tier: next-gen

      - alert: ExecutorClearDataplaneTaskRunningTooLong
        annotations:
          message: Runtime Executor task {{ $labels.task_name }} action {{ $labels.action_name }} is running too long on region {{ $labels.tidb_cluster_id }}.
        expr: |
          max by(executor_name, task_name, action_name, tidb_cluster_id) (
            runtime_executor_dag_action_running_walltime_seconds{action_name="clear-dataplane", result="running"}
          ) > 3600
        for: 1m
        labels:
          severity: major
          response_class: ticket
          component: cluster-ng-service
          oncall_team: premium
          tier: next-gen