groups:
  - name: dataflow-workflow
    rules:
      - alert: WorkflowDagInstanceDispatcherFailed
        annotations:
          message: dispatcher Dag Instance Failed
        expr: |
          sum(increase(fastflow_dispatcher_failed_total{worker_key=~"dataflow-workflow.*"}[1m])) > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: WorkflowDagInstanceParseFailed
        annotations:
          message: parse Dag Instance Failed
        expr: |
          sum(increase(fastflow_parser_parse_scheduled_dag_instance_failed_total{worker_key=~"dataflow-workflow.*"}[1m])) > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: RecoveryGroupWorkflowCriticalFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"recovery_group"} > 0
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: ReplicationLinkWorkflowCriticalFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"replication_link"} > 0
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: ReplicationLinkItemWorkflowCriticalFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"replication_link_item",business_action!~"delete_replication_link_item"} > 0
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: ReplicationLinkItemWorkflowFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"replication_link_item",business_action=~"delete_replication_link_item"} > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: RecoveryGroupFullBackupWorkflowCriticalFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"full_backup",business_action!~"delete_full_backup"} > 0
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: RecoveryGroupFullBackupWorkflowFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"full_backup",business_action=~"delete_full_backup"} > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: RecoveryGroupFullReplicationWorkflowCriticalFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"full_replication",business_action!~"delete_full_replication"} > 0
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: RecoveryGroupFullReplicationWorkflowFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"full_replication",business_action=~"delete_full_replication"} > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: TiCDCClusterWorkflowCriticalFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"ticdc_cluster",business_action!~"delete_ticdc_cluster"} > 0
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: TiCDCClusterWorkflowFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"ticdc_cluster",business_action=~"delete_ticdc_cluster"} > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: NodeGroupWorkflowCriticalFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"nodegroup",business_action!~"delete_nodegroup"} > 0
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: NodeGroupWorkflowFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"nodegroup",business_action=~"delete_nodegroup"} > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: BizPrivateLinkWorkflowFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"biz_private_link"} > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service
          oncall_team: tidb_x_tools
      - alert: BizChangefeedWorkflowFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow.*",business_type=~"biz_changefeed"} > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service    
          oncall_team: tidb_x_tools