groups:
  - name: mgmt.workflow
    rules:
      - alert: WorkflowLongDedicatedGlobal
        annotations:
          message: Dedicated-global workflow > 40m. tenant_id={{ $labels.tenant_id }} project_id={{ $labels.project_id }} biz_id={{ $labels.biz_id }} job={{ $labels.job }} dag={{ $labels.workflow_dag }} dag_ins_id={{ $labels.dag_ins_id }} component_type={{ $labels.component_type }} operation_type={{ $labels.operation_type }}
        expr: |
          (
            (
              max by (tenant_id, project_id, biz_id, workflow_dag, job, dag_ins_id, component_type, operation_type) (
                workflow_info{namespace=~"tidb-management-service", job="dedicated-global-service"}
              )
            )
            unless on (job, component_type, operation_type)
            (
              workflow_info{
                job="dedicated-global-service",
                component_type=~"tikv|tiflash",
                operation_type="node_scale_in"
              }
            )
          ) > 2400
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: dedicated_service

      - alert: ScaleInWorkflowLongDedicatedGlobal
        annotations:
          message: Dedicated-global scale-in workflow > 3h. tenant_id={{ $labels.tenant_id }} project_id={{ $labels.project_id }} biz_id={{ $labels.biz_id }} job={{ $labels.job }} dag={{ $labels.workflow_dag }} dag_ins_id={{ $labels.dag_ins_id }} component_type={{ $labels.component_type }} operation_type={{ $labels.operation_type }}
        expr: |
          max by (tenant_id, project_id, biz_id, workflow_dag, job, dag_ins_id, component_type, operation_type) (
            workflow_info{
              namespace=~"tidb-management-service",
              job="dedicated-global-service",
              component_type=~"tikv|tiflash",
              operation_type="node_scale_in"
            }
          ) > 10800
        for: 3m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: dedicated_service

      - alert: WorkflowLongMgmt
        annotations:
          message: TiDB mgmt workflow > 40m. tenant_id={{ $labels.tenant_id }} project_id={{ $labels.project_id }} biz_id={{ $labels.biz_id }} job={{ $labels.job }} dag={{ $labels.workflow_dag }} dag_ins_id={{ $labels.dag_ins_id }} component_type={{ $labels.component_type }} operation_type={{ $labels.operation_type }}
        expr: |
          max by (tenant_id, project_id, biz_id, workflow_dag, job, dag_ins_id, component_type, operation_type) (
            workflow_info{
              namespace=~"tidb-management-service",
              job="tidb-mgmt-service",
              workflow_dag!~"(dynamic_dag__(console_api_scale|mgmt_dedicated_create_migration_cluster).*|mgmt_nextgen_restore_cluster)", 
              o11y_cluster_info!="base/aws/us-west-2/39190b22-bb48-4167-a485-eec9917ac85a"
            }
          ) > 2400
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: WorkflowLongServerless
        annotations:
          message: Serverless workflow > 20m. tenant_id={{ $labels.tenant_id }} project_id={{ $labels.project_id }} biz_id={{ $labels.biz_id }} job={{ $labels.job }} dag={{ $labels.workflow_dag }} dag_ins_id={{ $labels.dag_ins_id }} component_type={{ $labels.component_type }} operation_type={{ $labels.operation_type }}
        expr: |
          max by (tenant_id, project_id, biz_id, workflow_dag, job, dag_ins_id, component_type, operation_type) (
            workflow_info{namespace=~"tidb-management-service", job="serverless-global-service", o11y_cluster_info!="base/aws/us-west-2/39190b22-bb48-4167-a485-eec9917ac85a"}
          ) > 1200
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: WorkflowLongNextgen
        annotations:
          message: Nextgen workflow > 2h. tenant_id={{ $labels.tenant_id }} biz_id={{ $labels.biz_id }} job={{ $labels.job }} dag={{ $labels.workflow_dag }} dag_ins_id={{ $labels.dag_ins_id }} component_type={{ $labels.component_type }} operation_type={{ $labels.operation_type }}
        expr: |
          max by (tenant_id, project_id, biz_id, workflow_dag, job, dag_ins_id, component_type, operation_type) (
            workflow_info{namespace=~"tidb-management-service", job="nextgen-global-service", workflow_dag!~"restore_.*"}
          ) > 7200
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control

      - alert: WorkflowRetryFailures
        annotations:
          message: Workflow retry failures > 10 in 5m. tenant_id={{ $labels.tenant_id }} project_id={{ $labels.project_id }} biz_id={{ $labels.biz_id }} job={{ $labels.job }} dag={{ $labels.workflow_dag }} dag_ins_id={{ $labels.dag_ins_id }} component_type={{ $labels.component_type }} operation_type={{ $labels.operation_type }}
        expr: |
          increase(workflow_watchdog_retried_dag_instance_total{namespace=~"tidb-management-service", retry_command_error="true"}[5m]) > 10
        for: 2m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control