# the customer alert rules for tiflow has been moved to rules/customer/data-migration.yaml
groups:
- name: data-migration
  rules:
  - alert: DMExecutorNotRunning
    annotations:
      summary: Data migration job executor not running for more than 1 minute
      message: Data migration job executor not running for more than 1 minute for data migration job
    expr: |
      max(dataflow_tiflow_server_master_job_num{status="Running"}) > count(dm_worker_task_state)
    for: 1m
    labels:
      severity: critical
      response_class: page
      component: data-migration
      oncall_team: tidb_x_tools
  - alert: DMMetErrorOnExport
    annotations:
      description: 'Data migration job <a href="https://TiDBCloudDomainName/console/clusters/{{ $labels.tidb_cluster_id }}/data-migration/{{ $labels.job_id }}">{{ $labels.job_id }}</a> of TiDB cluster TiDBClusterName meet some error during data export. Please check the error and see <a href="https://docs.pingcap.com/tidbcloud/tidb-cloud-dm-precheck-and-troubleshooting#migration-errors-and-solutions">Troubleshoot DM</a> for help.'
      summary: 'Data migration job hit an unrecoverable error during export'
    expr: |
      changes(dm_mydumper_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(project_id, job_id, source_id) increase(dm_mydumper_exit_with_error_count{resumable_err="true"}[2m]) > 3
    labels:
      severity: major
      response_class: ticket
      component: data-migration
      oncall_team: tidb_x_tools
      description: Data migration job met error during data export
      message: |
        Data migration job <a href="https://TiDBCloudDomainName/console/clusters/{{ $labels.tidb_cluster_id }}/data-migration/{{ $labels.job_id }}">{{ $labels.job_id }}</a> of TiDB cluster TiDBClusterName meet some error during data export. Please check the error and see <a href="https://docs.pingcap.com/tidbcloud/tidb-cloud-dm-precheck-and-troubleshooting#migration-errors-and-solutions">Troubleshoot DM</a> for help.
  - alert: DMMetErrorOnImport
    annotations:
      description: 'Data migration job <a href="https://TiDBCloudDomainName/console/clusters/{{ $labels.tidb_cluster_id }}/data-migration/{{ $labels.job_id }}">{{ $labels.job_id }}</a> of TiDB cluster TiDBClusterName meet some error during data import. Please check the error and see <a href="https://docs.pingcap.com/tidbcloud/tidb-cloud-dm-precheck-and-troubleshooting#migration-errors-and-solutions">Troubleshoot DM</a> for help.'
      summary: 'Data migration job hit an unrecoverable error during import'
    expr: |
      changes(dm_loader_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(project_id, job_id, source_id) increase(dm_loader_exit_with_error_count{resumable_err="true"}[2m]) > 3
    labels:
      severity: major
      response_class: ticket
      component: data-migration
      oncall_team: tidb_x_tools
      description: Data migration job met error during data import
      message: |
        Data migration job <a href="https://TiDBCloudDomainName/console/clusters/{{ $labels.tidb_cluster_id }}/data-migration/{{ $labels.job_id }}">{{ $labels.job_id }}</a> of TiDB cluster TiDBClusterName meet some error during data import. Please check the error and see <a href="https://docs.pingcap.com/tidbcloud/tidb-cloud-dm-precheck-and-troubleshooting#migration-errors-and-solutions">Troubleshoot DM</a> for help.
  - alert: DMMetErrorOnSync
    annotations:
      description: 'Data migration job <a href="https://TiDBCloudDomainName/console/clusters/{{ $labels.tidb_cluster_id }}/data-migration/{{ $labels.job_id }}">{{ $labels.job_id }}</a> of TiDB cluster TiDBClusterName meet some error during incremental data migration. Please check the error and see <a href="https://docs.pingcap.com/tidbcloud/tidb-cloud-dm-precheck-and-troubleshooting#migration-errors-and-solutions">Troubleshoot DM</a> for help.'
      summary: 'Data migration job hit an unrecoverable error during sync'
    expr: |
      rate(dm_syncer_exit_with_error_count{resumable_err="false"}[1m]) > 0 or on(project_id, job_id, source_id) increase(dm_syncer_exit_with_error_count{resumable_err="true"}[2m]) > 3
    labels:
      severity: warning
      component: data-migration
      oncall_team: tidb_x_tools
      description: Data migration job met error during incremental data migration
      message: |
        Data migration job <a href="https://TiDBCloudDomainName/console/clusters/{{ $labels.tidb_cluster_id }}/data-migration/{{ $labels.job_id }}">{{ $labels.job_id }}</a> of TiDB cluster TiDBClusterName meet some error during incremental data migration. Please check the error and see <a href="https://docs.pingcap.com/tidbcloud/tidb-cloud-dm-precheck-and-troubleshooting#migration-errors-and-solutions">Troubleshoot DM</a> for help.
  - alert: DMSyncPauseTooLong
    annotations:
      description: 'Data migration job <a href="https://TiDBCloudDomainName/console/clusters/{{ $labels.tidb_cluster_id }}/data-migration/{{ $labels.job_id }}">{{ $labels.job_id }}</a> of TiDB cluster TiDBClusterName has been paused for more than 6 hours during incremental migration, binlog in upstream database might be purged(depends on your database binlog purge strategy) and may cause incremental migration fail. See <a href="https://docs.pingcap.com/tidbcloud/tidb-cloud-dm-precheck-and-troubleshooting#migration-errors-and-solutions">Troubleshoot DM</a> for help.'
      summary: 'Data migration job has been paused for more than 6 hours'
    expr: |
      dm_worker_task_state{} == 3
    for: 6h
    labels:
      severity: major
      response_class: ticket
      component: data-migration
      oncall_team: tidb_x_tools
      message: |
        Data migration job <a href="https://TiDBCloudDomainName/console/clusters/{{ $labels.tidb_cluster_id }}/data-migration/{{ $labels.job_id }}">{{ $labels.job_id }}</a> of TiDB cluster TiDBClusterName has been paused for more than 6 hours during incremental migration, binlog in upstream database might be purged(depends on your database binlog purge strategy) and may cause incremental migration fail. See <a href="https://docs.pingcap.com/tidbcloud/tidb-cloud-dm-precheck-and-troubleshooting#migration-errors-and-solutions">Troubleshoot DM</a> for help.
      description: Data migration job has been paused for more than 6 hours during incremental migration
  - alert: DMReplicationLagTooLarge
    annotations:
      description: 'Replication lag is larger than {threshold} seconds and stilling increasing for more than {triggerInterval} minutes for data migration job <a href="https://TiDBCloudDomainName/console/clusters/{{ $labels.tidb_cluster_id }}/data-migration/{{ $labels.job_id }}">{{ $labels.job_id }}</a> of TiDB cluster TiDBClusterName. See <a href="https://docs.pingcap.com/tidbcloud/tidb-cloud-dm-precheck-and-troubleshooting#migration-errors-and-solutions">Troubleshoot DM</a> for help.'
      summary: 'Data migration replication lag exceeds 600 seconds and keeps increasing'
    expr: |
      delta(dm_syncer_replication_lag_gauge[1m]) > 0 and on(project_id, job_id, source_id) dm_syncer_replication_lag_gauge{} > 600
    for: 20m
    labels:
      severity: major
      response_class: ticket
      component: data-migration
      oncall_team: tidb_x_tools
      message: |
        Replication lag is larger than {threshold} seconds and stilling increasing for more than {triggerInterval} minutes for data migration job <a href="https://TiDBCloudDomainName/console/clusters/{{ $labels.tidb_cluster_id }}/data-migration/{{ $labels.job_id }}">{{ $labels.job_id }}</a> of TiDB cluster TiDBClusterName. See <a href="https://docs.pingcap.com/tidbcloud/tidb-cloud-dm-precheck-and-troubleshooting#migration-errors-and-solutions">Troubleshoot DM</a> for help.
      description: Replication lag is larger than {threshold} seconds and stilling increasing for more than {triggerInterval} minutes
  - alert: PremiumDMMigrationRecentFail
    annotations:
      description: 'Premium data migration task {{ $labels.task_id }} for cluster {{ $labels.cluster_id }} failed recently. Check premium DM task details and worker logs for the root cause.'
      summary: 'Premium data migration task failed recently'
    expr: |
      dataflow_premium_dm_recent_fail > 0
    for: 1m
    labels:
      severity: major
      response_class: ticket
      alerttype: event
      component: data-migration
      oncall_team: tidb_x_tools
      message: |
        Premium data migration task {{ $labels.task_id }} for cluster {{ $labels.cluster_id }} failed recently. Check premium DM task details and worker logs for the root cause.
      description: Premium data migration task failed within the recent 15 minute lookback window
  - alert: PremiumDMMigrationFailed
    annotations:
      description: 'Premium data migration for cluster {{ $labels.cluster_id }} has at least one task staying in failed state for more than 15 minutes. Check premium DM task details and worker logs for the root cause.'
      summary: 'Premium data migration task stays in failed state for more than 15 minutes'
    expr: |
      dataflow_premium_dm_task_status_count{stage="failed"} > 0
    for: 15m
    labels:
      severity: major
      response_class: ticket
      component: data-migration
      oncall_team: tidb_x_tools
      message: |
        Premium data migration for cluster {{ $labels.cluster_id }} has at least one task staying in failed state for more than 15 minutes. Check premium DM task details and worker logs for the root cause.
      description: Premium data migration has task(s) remaining in failed state