groups:
- name: ticdc-data-plane
  rules:
  - alert: BizChangefeedCheckpointHighDelay10minsNextGen
    expr: |
      ticdc_owner_checkpoint_ts_lag{tidb_cluster_id!~"^(10660386720259467020|10833157969408330931)$"} > 600
      and
      ticdc_owner_checkpoint_ts_lag{tidb_cluster_id!~"^(10660386720259467020|10833157969408330931)$"} offset 12h
    for: 5m # reduce noise for new created changefeed
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: ticdc-kernel
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: ticdc-kernel
      oncall_team: ticdc
    annotations:
      message: >
        Checkpoint of {{ $labels.changefeed }} delay more than 10 mins.
        TiDB Cluster: {{ $labels.tidb_cluster_id }}
        Reported by {{ $labels.instance }}
        Region: {{ $labels.seed_region }}
        Infra Arch: {{ $labels.seed_provider }}
  - alert: BizChangefeedCheckpointHighDelay3hNextGen
    expr: |
      ticdc_owner_checkpoint_ts_lag{tidb_cluster_id!~"^(10660386720259467020|10833157969408330931)$"} > 10800
      and
      ticdc_owner_checkpoint_ts_lag{tidb_cluster_id!~"^(10660386720259467020|10833157969408330931)$"} offset 12h
    for: 10m # reduce noise for new created changefeed
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: ticdc-kernel
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: ticdc-kernel
      oncall_team: ticdc
      source_tcoc: TCOC-2803
    annotations:
      message: >
        Checkpoint of {{ $labels.changefeed }} delay more than 3 hours.
        TiDB Cluster: {{ $labels.tidb_cluster_id }}
        Reported by {{ $labels.instance }}
        Region: {{ $labels.seed_region }}
        Infra Arch: {{ $labels.seed_provider }}
  - alert: BizChangefeedIsFailedByKernelStatusNextGen
    expr: ticdc_owner_status{tidb_cluster_id!~"^(10660386720259467020|10833157969408330931)$"} == 2
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: ticdc-kernel
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: ticdc-kernel
      oncall_team: ticdc
      source_tcoc: TCOC-2803
    annotations:
      message: >
        Changefeed: {{ $labels.changefeed }}
        TiDB Cluster: {{ $labels.tidb_cluster_id }}
        Instance: {{ $labels.instance }}
  - alert: BizChangefeedIsWarningByKernelStatusNextGen
    expr: ticdc_owner_status{tidb_cluster_id!~"^(10660386720259467020|10833157969408330931)$"} == 6
    for: 30m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: ticdc-kernel
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: ticdc-kernel
      oncall_team: ticdc
      source_tcoc: TCOC-2803
    annotations:
      message: >
        Changefeed: {{ $labels.changefeed }}
        TiDB Cluster: {{ $labels.tidb_cluster_id }}
        Instance: {{ $labels.instance }}
  - alert: TiKVSideCDCSinkMemoryHighUsageNextGen
    expr: sum(tikv_cdc_sink_memory_bytes{instance=~".*-tikv.*"}) by (tidb_cluster_id, instance) > 300000000
    for: 2m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: ticdc-kernel
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: ticdc-kernel
      oncall_team: ticdc
      source_tcoc: TCOC-2803
    annotations:
      message: >
        TiDB Cluster: {{ $labels.tidb_cluster_id }}
        Instance: {{ $labels.instance }}
        Usage of CDC sink memory in TiKV side is high which default is 512MB, please refer to SOP https://pingcap.feishu.cn/wiki/OTIewy2EoiAnGCkTezmc0xgHnfc to handle this case.
        Source_tcoc: TCOC-2803
  - alert: TiCDCInstanceFrequentlyRestart
    expr: |-
      sum by (tidb_cluster_id, instance, tenant, provider_type) (
          resets((-floor(process_start_time_seconds{component="ticdc"}))[1h:15s])
        and
          max_over_time(process_start_time_seconds{component="ticdc"}[30m] offset 1h) > 0
      ) >= 3
    for: 1m
    labels:
      tier: next-gen
      severity: critical
      response_class: page
      component: ticdc-kernel
      oncall_team: ticdc
    annotations:
      description: 'TiCDC {{ $labels.instance }} in cluster {{ $labels.tidb_cluster_id }} has restarted {{ $value }} times in the past hour. Please investigate the root cause and ensure TiCDC server stability. cluster: {{ $labels.tidb_cluster_id }}, instance: {{ $labels.instance }}, value: {{ $value }}'
      summary: TiCDC servers in a cluster restarted {{ $value }} times in past an hour
  - alert: TiCDCLogPullerMemoryQuotaHighWithCluster
    expr: |-
      sum(ticdc_dynamic_stream_memory_usage{k8s_cluster=~"$k8s_cluster", sharedpool_id="$tidb_cluster", instance=~"$ticdc_instance", module=~"log-puller", tenant=~".*1372813089209289635"}) by (instance, type) > 100
    for: 5m
    labels:
      tier: next-gen
      severity: major
      response_class: ticket
      component: ticdc
      stability_governance: next-gen-kernel-tcocp
      visibility: internal
      source_component: ticdc-kernel
      oncall_team: tidb_x_tools
      source_tcoc: TCOC-2803
    annotations:
      message: >
        TiCDC log puller memory quota is high in cluster {{ $labels.cluster_id }}. Current value: {{ $value }}%