groups:
- name: ticdc
  rules:
  - alert: BizChangefeedCheckpointHighDelay1h
    # filter out BIG3 tenants, since they have special rule
    # filter out cluster: eagle-insight(mk-jp-tidb-gcp, it's a dev cluster and its kafka always down)
    # filter out cluster: taka-h-ticdc-verification(mk-jp-tidb-gcp, it's a dev cluster)
    # filter out cluster: production (Fujitsu Japan, burst traffic, flush configurations are not efficient for replication, but cost-effective)
    expr: ticdc_owner_checkpoint_ts_lag{tenant!~"1372813089209061633|1372813089209238420|1372813089209256793|1372813089209261054", tidb_cluster_id!~"1379661944646413949|1379661944646225237|10187977631706938951"} > 3600 and on (tenant, project, tidb_cluster_id, changefeed) ticdc_owner_checkpoint_ts_lag{} offset 1h
    for: 10m # reduce noise for new created changefeed
    labels:
      severity: critical
      response_class: page
      expr: ticdc_owner_checkpoint_ts_lag{} > 1h
      component: ticdc
      oncall_team: resilience
    annotations:
      message: >
        Checkpoint of {{ $labels.changefeed }} delay more than 1 hour.
        TiDB Cluster: {{ $labels.tidb_cluster_id }}
        Reported by {{ $labels.instance }}
        Region: {{ $labels.seed_region }}
        Infra Arch: {{ $labels.seed_provider }}
  - alert: BizChangefeedCheckpointHighDelay3h
    # for cluster: us-trans-prod (wetech, it sometimes has too high traffic, and the customer can tolerate the lag of changefeeed)
    # for cluster: production (Fujitsu Japan, burst traffic, flush configurations are not efficient for replication, but cost-effective)
    expr: ticdc_owner_checkpoint_ts_lag{tidb_cluster_id=~"1379661944646416107|1379661944646225237"} > 10800 and on (tenant, project, tidb_cluster_id, changefeed) ticdc_owner_checkpoint_ts_lag{} offset 1h
    for: 10m # reduce noise for new created changefeed
    labels:
      severity: major
      response_class: ticket
      expr: ticdc_owner_checkpoint_ts_lag{} > 3h
      component: ticdc
      oncall_team: resilience
    annotations:
      message: >
        Checkpoint of {{ $labels.changefeed }} delay more than 3 hours.
        TiDB Cluster: {{ $labels.tidb_cluster_id }}
        Reported by {{ $labels.instance }}
        Region: {{ $labels.seed_region }}
        Infra Arch: {{ $labels.seed_provider }}
  # Changefeed status shows on the web console
  # Only BIG3 need these internal alert, and they have special rule
  # - alert: BizChangefeedIsFailedByKernelStatus
  # - alert: BizChangefeedIsWarningByKernelStatus
  - alert: RGChangefeedCheckpointHighDelay10m
    expr: ticdc_owner_checkpoint_ts_lag{instance=~"^rg.*"}  > 600
    for: 15m # reduce noise for traffic spikes and temporary errors
    labels:
      severity: critical
      response_class: page
      expr: ticdc_owner_checkpoint_ts_lag{instance=~"^rg.*"} > 10m
      component: ticdc
      oncall_team: tidb_x_tools
    annotations:
      message: >
        Checkpoint of {{ $labels.changefeed }} delay more than 10 minutes.
        TiDB Cluster: {{ $labels.tidb_cluster_id }}
        Reported by {{ $labels.instance }}
        Region: {{ $labels.seed_region }}
        Infra Arch: {{ $labels.seed_provider }}
  - alert: RGChangefeedTiCDCNodeFrequentlyUnavailable
    expr: sum(changes(floor(process_start_time_seconds{instance=~"^rg.*", component="ticdc"})[1h:15s])) by (cluster_id, instance) >= 4
    for: 1m
    labels:
      severity: critical
      response_class: page
      expr: sum(changes(floor(process_start_time_seconds{instance=~"^rg.*", component="ticdc"})[1h:15s])) by (cluster_id, instance) >= 4
      component: ticdc
      oncall_team: tidb_x_tools
    annotations:
      message: >
        CDC pod restart frequently.
        TiDB Cluster: {{ $labels.cluster_id }}
        Instance: {{ $labels.instance }}
        Tenant: {{ $labels.tenant }}
  - alert: RGChangefeedIsFailedByKernelStatus
    expr: ticdc_owner_status{instance=~"^rg.*"} == 2
    labels:
      severity: critical
      response_class: page
      expr: ticdc_owner_status{instance=~"^rg.*"} == 2
      component: ticdc
      oncall_team: tidb_x_tools
    annotations:
      message: >
        Changefeed: {{ $labels.changefeed }}
        TiDB Cluster: {{ $labels.cluster_id }}
        Instance: {{ $labels.instance }}
  - alert: RGChangefeedIsWarningByKernelStatus
    expr: ticdc_owner_status{instance=~"^rg.*"} == 6
    for: 2m
    labels:
      severity: major
      response_class: ticket
      expr: ticdc_owner_status{instance=~"^rg.*"} == 6
      component: ticdc
      oncall_team: tidb_x_tools
    annotations:
      message: >
        Changefeed: {{ $labels.changefeed }}
        TiDB Cluster: {{ $labels.cluster_id }}
        Instance: {{ $labels.instance }}
  - alert: TiKVSideCDCSinkMemoryHighUsage
    # filter out BIG3 tenants, since they have special rule
    expr: sum(tikv_cdc_sink_memory_bytes{instance=~".*-tikv.*", tenant!~"1372813089209061633|1372813089209238420|1372813089209256793|1372813089209261054"}) by (cluster_id, instance) > 300000000
    for: 2m
    labels:
      severity: critical
      response_class: page
      expr: sum(tikv_cdc_sink_memory_bytes{instance=~".*-tikv.*", tenant!~"1372813089209061633|1372813089209238420|1372813089209256793|1372813089209261054"}) by (cluster_id, instance) > 300000000
      component: ticdc
      oncall_team: resilience
    annotations:
      message: >
        TiDB Cluster: {{ $labels.cluster_id }}
        Instance: {{ $labels.instance }}
        Usage of CDC sink memory in TiKV side is high which default is 512MB, please refer to SOP https://pingcap.feishu.cn/wiki/OTIewy2EoiAnGCkTezmc0xgHnfc to handle this case.

# Those alerts move to rules/dedicated/swat/data-plane/kernel-tcocp.yaml
# Special rules for specific customers, which will alert both cloud team and kernel team
# one problem will have two alert rule: one for cloud team, one for kernel team
# 1. alert for tenant: wetech(1372813089209061633)
# 2. filter out test project: aws-pressure-test-saas(wetech: 1372813089454536384)