groups:
- name: alert.rules
  rules:
  - alert: DM_master_all_down
    expr: up{job="dm_master"} == 0
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: up{job="dm_master"} == 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: DM master all down, metrics not relyable

  - alert: DM_remain_storage_of_relay_log
    expr: dm_relay_space{type="available"} < 10*1024*1024*1024
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: dm_relay_space{type="available"} < 10*1024*1024*1024
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: DM remain storage of relay log

  - alert: DM_relay_process_exits_with_error
    expr: changes(dm_relay_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(instance, job) increase(dm_relay_exit_with_error_count{resumable_err="true"}[2m]) > 3
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: changes(dm_relay_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(instance, job) increase(dm_relay_exit_with_error_count{resumable_err="true"}[2m]) > 3
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: DM relay process exits with error

  - alert: DM_relay_log_data_corruption
    expr: changes(dm_relay_data_corruption[1m]) > 0
    labels:
      env: ENV_LABELS_ENV
      level: emergency
      expr: changes(dm_relay_data_corruption[1m]) > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: DM relay log data corruption

  - alert: DM_fail_to_read_binlog_from_master
    expr: changes(dm_relay_read_error_count[1m]) > 0
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: changes(dm_relay_read_error_count[1m]) > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: DM fail to read binlog from master

  - alert: DM_fail_to_write_relay_log
    expr: changes(dm_relay_write_error_count[1m]) > 0
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: changes(dm_relay_write_error_count[1m]) > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: DM fail to write relay log

  - alert: DM_dump_process_exists_with_error
    expr: changes(dm_mydumper_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(source_id, task) increase(dm_mydumper_exit_with_error_count{resumable_err="true"}[2m]) > 3
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: changes(dm_mydumper_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(source_id, task) increase(dm_mydumper_exit_with_error_count{resumable_err="true"}[2m]) > 3
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, task: {{ $labels.task }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: DM dump process exists with error

  - alert: DM_load_process_exists_with_error
    expr: changes(dm_loader_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(source_id, task) increase(dm_loader_exit_with_error_count{resumable_err="true"}[2m]) > 3
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: changes(dm_loader_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(source_id, task) increase(dm_loader_exit_with_error_count{resumable_err="true"}[2m]) > 3
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, task: {{ $labels.task }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: DM load process exists with error

  - alert: DM_sync_process_exists_with_error
    expr: changes(dm_syncer_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(source_id, task) increase(dm_syncer_exit_with_error_count{resumable_err="true"}[2m]) > 3
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: changes(dm_syncer_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(source_id, task) increase(dm_syncer_exit_with_error_count{resumable_err="true"}[2m]) > 3
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, task: {{ $labels.task }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: DM sync process exists with error

  - alert: DM_task_state
    expr: dm_worker_task_state == 3
    for: 20m
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: dm_worker_task_state == 3
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, task: {{ $labels.task }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: dm worker paused exceed 20 min

  - alert: DM_binlog_file_gap_between_master_relay
    expr: dm_relay_binlog_file{node="master"} - ON(instance, job) dm_relay_binlog_file{node="relay"} > 1
    for: 10m
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: dm_relay_binlog_file{node="master"} - ON(instance, job) dm_relay_binlog_file{node="relay"} > 1
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, task: {{ $labels.task }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: dm relay binlog file not catch up master server exceed 10 min

  - alert: DM_binlog_file_gap_between_master_syncer
    expr: dm_syncer_binlog_file{node="master"} - ON(instance, task, job) dm_syncer_binlog_file{node="syncer"} > 1
    for: 10m
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: dm_syncer_binlog_file{node="master"} - ON(instance, task, job) dm_syncer_binlog_file{node="syncer"} > 1
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, task: {{ $labels.task }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: dm syncer binlog file not catch up master server exceed 10 min

  - alert: DM_binlog_file_gap_between_relay_syncer
    expr: dm_relay_binlog_file{node="relay"} - ON(instance, job) group_right dm_syncer_binlog_file{node="syncer"} > 1
    for: 10m
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: dm_relay_binlog_file{node="relay"} - ON(instance, job) group_right dm_syncer_binlog_file{node="syncer"} > 1
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, task: {{ $labels.task }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: dm syncer binlog file not catch up relay exceed 10 min

  - alert: DM_worker_offline
    expr: dm_master_worker_state == 0
    for: 1h
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: dm_master_worker_state == 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, task: {{ $labels.task }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: dm worker offline exceed 1h

  - alert: DM_pending_DDL
    expr: dm_master_ddl_state_number > 0
    for: 1h
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: dm_master_ddl_state_number > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, task: {{ $labels.task }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: DDL pending exceed 1h

  - alert: DM_DDL_error
    expr: increase(dm_master_shard_ddl_error[1m]) > 0
    labels:
      env: ENV_LABELS_ENV
      level: critical
      expr: increase(dm_master_shard_ddl_error[1m]) > 0
    annotations:
      description: 'cluster: ENV_LABELS_ENV, instance: {{ $labels.instance }}, task: {{ $labels.task }}, values: {{ $value }}'
      value: '{{ $value }}'
      summary: DDL error happens