groups:
  - name: nextgen-dataflow-worker-errorlog
    interval: 30s
    rules:
      - alert: DataflowWorkerKernelChangefeedNotFound
        expr: |
          sum by (app, ticdc_cluster_id, kernel_changefeed_id) (
            count_over_time(
              {app=~"dataflow-worker|dataflow-ng-worker", namespace="prod-ms"}
                |= `kernel changefeed is not found in changefeed entities`
                | regexp `\[ticdc_cluster_id=(?P<ticdc_cluster_id>[^\]]+)\]`
                | regexp `\[kernel_changefeed_id=(?P<kernel_changefeed_id>[^\]]+)\]`
              [5m]
            )
          ) > 3
        for: 2m
        labels:
          severity: critical
          response_class: page
          tier: next-gen
          component: ticdc
          oncall_team: tidb_x_tools
        annotations:
          summary: "dataflow worker reports kernel changefeed missing from entities"
          message: 'app {{ $labels.app }} reported kernel changefeed {{ $labels.kernel_changefeed_id }} (ticdc_cluster_id={{ $labels.ticdc_cluster_id }}) is not found in changefeed entities; entity and kernel state may be out of sync'
          runbook_url: https://github.com/tidbcloud/runbooks/wiki/DataflowWorkerKernelChangefeedNotFound