groups:
  # Alerts on the data pipeline orchestration in dataflow-service-ng. A pipeline
  # drives three things it does not own - a snapshot export, a TiCDC changefeed
  # and a TiDB Lake integration task - so the interesting failures are the ones
  # where the pipeline's own row still looks fine and one of the three has
  # stopped, or where the observer that keeps that row honest has itself died.
  #
  # Series come from the dataflow-ng-worker collector; the job filter keeps the
  # workflow binary, which registers the same collector, from doubling every
  # alert.
  #
  # The per-pipeline rules below are not aggregated because that worker is a
  # singleton (replicas: 1 in app-delivery, base and both envs). The collector
  # reads the database at scrape time, so a second replica would emit identical
  # series differing only in pod, and every one of these would notify twice -
  # if the worker is ever scaled, these expressions need a max by (...) around
  # them.
  - name: data-pipeline
    rules:
      - alert: DataPipelineRecentlyFailed
        annotations:
          summary: Data pipeline failed
          message: Data pipeline {{ $labels.pipeline_id }} on cluster {{ $labels.cluster_id }} failed (error_code={{ $labels.error_code }}). Check the pipeline's last error and the export, changefeed and Lake task it owns.
          description: A data pipeline entered the failed state within the recent 15 minute lookback window
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/cluster-next-gen/global/data-pipeline.rules.yaml
        expr: |
          dataflow_data_pipeline_recent_fail{job="dataflow-ng-worker"} > 0
        for: 1m
        labels:
          severity: major
          response_class: ticket
          alerttype: event
          component: data-pipeline
          oncall_team: tidb_x_tools
          tier: next-gen

      - alert: DataPipelineStuckInFailedState
        annotations:
          summary: Data pipeline left in the failed state
          message: Data pipeline {{ $labels.pipeline_id }} on cluster {{ $labels.cluster_id }} has been failed for {{ printf "%.0f" $value }} seconds. Nothing will retry it; it needs a decision.
          description: A data pipeline has stayed in the failed state long after the failure event fired
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/cluster-next-gen/global/data-pipeline.rules.yaml
        # Per-pipeline age rather than a count with `for: 30m`: a count that
        # stays non-zero because failures keep arriving and being cleared would
        # satisfy `for` without any one pipeline having sat there at all, and
        # the message would be claiming something nobody could verify.
        expr: |
          dataflow_data_pipeline_status_duration_seconds{job="dataflow-ng-worker",status="failed"} > 30 * 60
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: data-pipeline
          oncall_team: tidb_x_tools
          tier: next-gen

      - alert: DataPipelineCreatingTooLong
        annotations:
          summary: Data pipeline creating for too long
          message: Data pipeline {{ $labels.pipeline_id }} on cluster {{ $labels.cluster_id }} has been creating for more than 30 minutes. Check the create DAG and whether the export, changefeed and Lake resources were all provisioned.
          description: Data pipeline creation has not completed within 30 minutes
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/cluster-next-gen/global/data-pipeline.rules.yaml
        expr: |
          dataflow_data_pipeline_status_duration_seconds{job="dataflow-ng-worker",status="creating"} > 30 * 60
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: data-pipeline
          oncall_team: tidb_x_tools
          tier: next-gen

      - alert: DataPipelineCreatingTooLongCritical
        annotations:
          summary: Data pipeline creating for far too long
          message: Data pipeline {{ $labels.pipeline_id }} on cluster {{ $labels.cluster_id }} has been creating for more than 60 minutes.
          description: Data pipeline creation has not completed within 60 minutes
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/cluster-next-gen/global/data-pipeline.rules.yaml
        expr: |
          dataflow_data_pipeline_status_duration_seconds{job="dataflow-ng-worker",status="creating"} > 60 * 60
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: data-pipeline
          oncall_team: tidb_x_tools
          tier: next-gen

      # Pause, resume, edit and delete are short operations driven by a DAG.
      # Half an hour in any of them means the DAG is wedged, not slow.
      - alert: DataPipelineTransitionStuck
        annotations:
          summary: Data pipeline stuck mid-transition
          message: Data pipeline {{ $labels.pipeline_id }} on cluster {{ $labels.cluster_id }} has been in {{ $labels.status }} for more than 30 minutes. Check the corresponding workflow DAG instance.
          description: A data pipeline lifecycle transition has not completed within 30 minutes
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/cluster-next-gen/global/data-pipeline.rules.yaml
        expr: |
          dataflow_data_pipeline_status_duration_seconds{job="dataflow-ng-worker",status=~"pausing|resuming|editing|deleting"} > 30 * 60
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: data-pipeline
          oncall_team: tidb_x_tools
          tier: next-gen

      # Every other alert here reads what the observer last wrote, so a dead
      # observer leaves a broken pipeline looking exactly as healthy as it did
      # when we lost sight of it. This is the alert that notices.
      #
      # Creating is excluded. An observation counts as successful only once
      # every child exists and answers, and a never-observed pipeline is
      # measured from creation - so a create that legitimately takes a while,
      # which a large export does, would trip this before the pipeline has
      # done anything wrong. Creation age is what the two CreatingTooLong
      # rules below measure, and they measure it directly.
      - alert: DataPipelineObservationStale
        annotations:
          summary: Data pipeline has not been observed successfully
          message: Data pipeline {{ $labels.pipeline_id }} on cluster {{ $labels.cluster_id }} has had no successful observation for {{ printf "%.0f" $value }} seconds. Its reported status, lag and child errors are all that stale.
          description: No successful observation of a data pipeline's children for more than 15 minutes
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/cluster-next-gen/global/data-pipeline.rules.yaml
        expr: |
          dataflow_data_pipeline_observation_staleness_seconds{job="dataflow-ng-worker",status!="creating"} > 15 * 60
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: data-pipeline
          oncall_team: tidb_x_tools
          tier: next-gen

      # If the least stale pipeline in the fleet is stale, no pipeline is being
      # observed at all: the worker loop is down rather than one pipeline's
      # children being unreachable.
      - alert: DataPipelineObserverStalled
        annotations:
          summary: Data pipeline observer is not running
          message: No data pipeline anywhere has been observed successfully in the last 10 minutes. The observation worker is down or failing for every pipeline; all data pipeline status is unreliable until it recovers.
          description: The data pipeline observation worker has produced no successful observation fleet-wide
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/cluster-next-gen/global/data-pipeline.rules.yaml
        expr: |
          min(dataflow_data_pipeline_observation_staleness_seconds{job="dataflow-ng-worker"}) > 10 * 60
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: data-pipeline
          oncall_team: tidb_x_tools
          tier: next-gen

      # The divergence case: the pipeline row still says running, but one of the
      # three things it orchestrates is reporting an error.
      - alert: DataPipelineChildError
        annotations:
          summary: Data pipeline child resource reporting an error
          message: The {{ $labels.child }} of data pipeline {{ $labels.pipeline_id }} on cluster {{ $labels.cluster_id }} is reporting an error while the pipeline is still supposed to be running.
          description: A data pipeline's export, changefeed or Lake task is in error while the pipeline is live
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/cluster-next-gen/global/data-pipeline.rules.yaml
        expr: |
          dataflow_data_pipeline_child_error{job="dataflow-ng-worker"} > 0
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: data-pipeline
          oncall_team: tidb_x_tools
          tier: next-gen

      # The largest end-to-end budget a user can configure is 900s (600s TiCDC
      # flush plus 300s destination poll), so 1800s is twice the worst legal
      # setting and cannot fire on a correctly configured pipeline that is
      # keeping up.
      - alert: DataPipelineDeliveryLagTooHigh
        annotations:
          summary: Data pipeline delivery lag too high
          message: Data pipeline {{ $labels.pipeline_id }} on cluster {{ $labels.cluster_id }} is {{ printf "%.0f" $value }} seconds behind between the changefeed checkpoint and the destination's resolved position.
          description: Data pipeline end-to-end delivery lag has exceeded 30 minutes
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/cluster-next-gen/global/data-pipeline.rules.yaml
        expr: |
          dataflow_data_pipeline_latency_seconds{job="dataflow-ng-worker"} > 30 * 60
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: data-pipeline
          oncall_team: tidb_x_tools
          tier: next-gen