groups:
  - name: dataflow-service
    rules:
      - alert: DataflowNGPodRestartsFrequently
        annotations:
          summary: The dataflow-service ng pod restart frequently
          message: Container {{ $labels.namespace }}/{{ $labels.container }} is restarting {{ printf "%.2f" $value }} times in 30 minutes.
        expr: |
          increase(kube_pod_container_status_restarts_total{container=~"dataflow-ng.*"}[30m]) > 1
        for: 10m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service-ng
          oncall_team: tidb_x_tools
          tier: next-gen
      - alert: DataflowNGGrpcErrorQPSTooHigh
        annotations:
          summary: The dataflow-service ng grpc error qps too high
          message: GRPC Service {{ $labels.grpc_method }}, GRPC Method{{ $labels.grpc_method }} error qps too high in 5 minutes.
        expr: |
          sum(rate(grpc_server_handled_total{namespace=~"prod-ms", pod=~"dataflow-ng.*", grpc_service=~"dataflow-ng.*", grpc_type="unary", grpc_code!="OK"}[5m])) by (grpc_service, grpc_method, grpc_code) > 0.1
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service-ng
          oncall_team: tidb_x_tools
          tier: next-gen
  - name: dataflow-workflow
    rules:
      - alert: WorkflowNGDagInstanceDispatcherFailed
        annotations:
          message: dispatcher Dag Instance Failed
        expr: |
          sum(increase(fastflow_dispatcher_failed_total{worker_key=~"dataflow-workflow-ng.*"}[1m])) > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service-ng
          oncall_team: tidb_x_tools
          tier: next-gen
      - alert: WorkflowDagInstanceParseFailed
        annotations:
          message: parse Dag Instance Failed
        expr: |
          sum(increase(fastflow_parser_parse_scheduled_dag_instance_failed_total{worker_key=~"dataflow-workflow-ng.*"}[1m])) > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service-ng
          oncall_team: tidb_x_tools
          tier: next-gen
      - alert: BizChangefeedNgWorkflowFailed
        annotations:
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow-ng.*",business_type=~"biz_changefeed"} > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service-ng
          oncall_team: tidb_x_tools
          tier: next-gen
      - alert: DataPipelineNgWorkflowFailed
        annotations:
          summary: A data pipeline workflow DAG failed
          message: The Workflow {{ $labels.dag_ins_id }} is Failed. business_type is {{ $labels.business_type }}, business_action is {{ $labels.business_action }}, business_id is {{ $labels.business_id }}
        expr: |
          fastflow_executor_dag_ins_failed{worker_key=~"dataflow-workflow-ng.*",business_type=~"data_pipeline"} > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: data-pipeline
          oncall_team: tidb_x_tools
          tier: next-gen
      - alert: ChangefeedCreationFailed
        annotations:
          summary: Changefeed creation failed due to a system error
          message: Changefeed {{ $labels.changefeed_id }} on cluster {{ $labels.cluster_id }} failed to create (err_type={{ $labels.err_type }}, message={{ $labels.message }}).
        expr: |
          increase(dataflow_ticdc_changefeed_create_result_total{result="failed",err_type="SystemError",provider_type="nextgen"}[5m]) > 0
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service-ng
          oncall_team: tidb_x_tools
          tier: next-gen
      - alert: ChangefeedCreationFailedUserError
        annotations:
          summary: Changefeed creation failed due to user configuration error
          message: Changefeed {{ $labels.changefeed_id }} on cluster {{ $labels.cluster_id }} failed to create due to user error (err_type={{ $labels.err_type }}, message={{ $labels.message }}).
        expr: |
          increase(dataflow_ticdc_changefeed_create_result_total{result="failed",err_type!="SystemError",provider_type="nextgen"}[5m]) > 0
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service-ng
          oncall_team: tidb_x_tools
          tier: next-gen
  - name: dataflow-worker
    rules:
      - alert: TiDBClusterChangefeedCreatingDurationTooLong
        annotations:
          message: The TiDB Cluster {{ $labels.cluster_id }}/{{ $labels.changefeed_id }} changefeed in creating status more than 30m.
        expr: |
          avg by (cluster_id, changefeed_id)(dataflow_changefeed_status{job="dataflow-ng-worker"}) == 0
        for: 30m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service-ng
          oncall_team: tidb_x_tools
          tier: next-gen
      - alert: TiDBClusterChangefeedCreatingDurationTooLongCritical
        annotations:
          message: The TiDB Cluster {{ $labels.cluster_id }}/{{ $labels.changefeed_id }} changefeed in creating status more than 50m.
        expr: |
          avg by (cluster_id, changefeed_id)(dataflow_changefeed_status{job="dataflow-ng-worker"}) == 0
        for: 50m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service-ng
          oncall_team: tidb_x_tools
          tier: next-gen
      - alert: TiDBClusterChangefeedScalingDurationTooLong
        annotations:
          message: The TiDB Cluster {{ $labels.cluster_id }}/{{ $labels.changefeed_id }} changefeed in scaling status more than 30m.
        expr: |
          avg by (cluster_id, changefeed_id)(dataflow_changefeed_status{job="dataflow-ng-worker"}) == 9
        for: 30m
        labels:
          severity: major
          response_class: ticket
          component: dataflow-service-ng
          oncall_team: tidb_x_tools
          tier: next-gen
      - alert: TiDBClusterChangefeedScalingDurationTooLongCritical
        annotations:
          message: The TiDB Cluster {{ $labels.cluster_id }}/{{ $labels.changefeed_id }} changefeed in scaling status more than 50m.
        expr: |
          avg by (cluster_id, changefeed_id)(dataflow_changefeed_status{job="dataflow-ng-worker"}) == 9
        for: 50m
        labels:
          severity: critical
          response_class: page
          component: dataflow-service-ng
          oncall_team: tidb_x_tools
          tier: next-gen