groups:
- name: alert.rules
  rules:
  # server related alter rules
  - alert: cdc_multiple_owners
    expr: sum(rate(ticdc_owner_ownership_counter[240s])) >= 2
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr: sum(rate(ticdc_owner_ownership_counter[240s])) >= 2
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: cdc cluster has multiple owners

  - alert: cdc_no_owner
    expr: sum(rate(ticdc_owner_ownership_counter[240s])) < 0.5
    for: 10m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr: sum(rate(ticdc_owner_ownership_counter[240s])) < 0.5
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: cdc cluster has no owner for more than 10 minutes

  # changefeed related alter rules
  - alert: ticdc_changefeed_failed
    expr: (max_over_time(ticdc_owner_status[1m]) == 2) > 0
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: (max_over_time(ticdc_owner_status[1m]) == 2) > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: cdc changefeed failed, it can not be automatically resumed

  - alert: cdc_checkpoint_high_delay
    expr: ticdc_owner_checkpoint_ts_lag > 600
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: ticdc_owner_checkpoint_ts_lag > 600
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: cdc owner checkpoint delay more than 10 minutes

  - alert: cdc_resolvedts_high_delay
    expr: ticdc_owner_resolved_ts_lag > 300
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: ticdc_owner_resolved_ts_lag > 300
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: cdc owner resolved ts delay more than 5 minutes

  - alert: ticdc_sink_execution_error
    expr: changes(ticdc_sink_execution_error[1m]) > 0
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr: changes(ticdc_sink_execution_error[1m]) > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: cdc sink execution meets errors

  - alert: ticdc_processor_exit_with_error_count
    expr: changes(ticdc_processor_exit_with_error_count[1m]) > 0
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr: changes(ticdc_processor_exit_with_error_count[1m]) > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: cdc processor exits with error

  - alert: ticdc_changefeed_meet_error
    expr: (max_over_time(ticdc_owner_status[1m]) == 1) > 0
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr: (max_over_time(ticdc_owner_status[1m]) == 1) > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: cdc changefeed meet error
  
  # tikv related alter rules
  - alert: tikv_cdc_min_resolved_ts_no_change_for_1m
    expr: changes(tikv_cdc_min_resolved_ts[1m]) < 1 and ON (instance) tikv_cdc_region_resolve_status{status="resolved"} > 0 and ON (instance) tikv_cdc_captured_region_total > 0
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr: changes(tikv_cdc_min_resolved_ts[1m]) < 1 and ON (instance) tikv_cdc_region_resolve_status{status="resolved"} > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $labels.instance }}'
      summary: tikv cdc min resolved ts no change for 1m

  - alert: tikv_cdc_scan_duration_seconds_more_than_10min
    expr: histogram_quantile(0.9, rate(tikv_cdc_scan_duration_seconds_bucket{}[1m])) > 600
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr: histogram_quantile(0.9, rate(tikv_cdc_scan_duration_seconds_bucket{}[1m])) > 600
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: tikv cdc scan duration seconds more than 10 min

  - alert: ticdc_memory_abnormal
    expr: go_memstats_heap_alloc_bytes{job="ticdc"} > 1e+10
    for: 1m
    labels:
      env: ENV_LABELS_ENV
      level: warning
      expr: go_memstats_heap_alloc_bytes{job="ticdc"} > 1e+10
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values:{{ $value }}'
      value: '{{ $value }}'
      summary: TiCDC heap memory usage is over 10 GB