groups:
- name: backup-restore
  rules:
  - alert: TiDBClusterBackupDurationTooLong
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} backup running duration more than 20h
    expr: |
      dataflow_backup_execution_status_duration_seconds{status="running", cluster_id!~"^(1379661944646415110|1379661944646413605|1379661944646414132|1379661944646413035)$"} > 60 * 60 * 20
    labels:
      severity: major
      response_class: ticket
      component: br
      oncall_team: tidb_x_tools
  - alert: TiDBClusterBackupDurationTooLongCritical
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} backup running duration more than 24h
    expr: |
      dataflow_backup_execution_status_duration_seconds{status!~"failed|success|skipped|terminated", cluster_id!~"^(1379661944646415110|1379661944646413605|1379661944646414132|1379661944646413035|10970866242657397869)$"} > 60 * 60 * 24
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  - alert: BitgetWalletTiDBClusterBackup16h
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} backup running duration more than 16h
    expr: |
      dataflow_backup_execution_status_duration_seconds{status="running", cluster_id="1379661944646413605"} > 60 * 60 * 16
    labels:
      severity: major
      response_class: ticket
      component: br
      oncall_team: tidb_x_tools
  - alert: TiDBClusterBackupMoreThan24Hours
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} backup running duration more than 24h. There may be critical kernel problem, please look into it.
    expr: |
      dataflow_backup_execution_status_duration_seconds{status="running"} > 60 * 60 * 24
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  - alert: TiDBClusterBackupStatusFailed
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} {{ $labels.backup_type }} backup status is failed.
    expr: |
      dataflow_backup_execution_status_duration_seconds{status="failed"} > 0
    labels:
      severity: critical
      response_class: page
      alerttype: event
      component: br
      oncall_team: tidb_x_tools
  - alert: TiDBClusterBackupStatusUnknown
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} backup in unknown status more than 15m.
        backup record created in central database but backup custom resource can not created. please check shoot apiserver status.
    expr: |
      dataflow_backup_execution_status_duration_seconds{status="unknown"} > 60 * 15
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  # backup pending exclude clusters of Colopl on gardener arch
  - alert: TiDBClusterBackupStatusPending
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} backup in pending status more than 1h.
    expr: |
      (dataflow_backup_execution_status_duration_seconds{status="pending",cluster_id!~"^(1379661944581754072|1379661944629154076|1379661944593934067)$"}) > 60 * 60
    labels:
      severity: major
      response_class: ticket
      component: br
      oncall_team: tidb_x_tools
  - alert: AutoBackupScheduledFailedMoreThan2Times
    annotations:
      message: The auto backup schedule failed more than 2 times last hour.
      runbook_url: https://clinic.pingcap.com/grafana/d/QCJCPPKVz/dataflow-service-monitor?orgId=1&viewPanel=124&from=now-3h&to=now
    expr: |
      increase(dataflow_worker_backup_scheduler_abnormal{type="auto_backup"}[1h]) > 2
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  - alert: BackupFailedToBeCleanedUp
    annotations:
      message: The expired {{ $labels.cluster_id }}/{{ $labels.backup_id }} are not cleaned up.
    expr: |
      dataflow_worker_backup_clean_up_abnormal == 1
    labels:
      severity: major
      response_class: ticket
      component: br
      oncall_team: tidb_x_tools
  # log backup scheculed/pending/stopping
  - alert: TiDBClusterLogBackupStatusScheduled
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} log backup in scheduled status more than 1h.
    expr: |
      (dataflow_log_backup_execution_status_duration_seconds{status="scheduled"}) > 60 * 60
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  - alert: TiDBClusterLogBackupStatusPending
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} log backup in pending status more than 1h.
    expr: |
      (dataflow_log_backup_execution_status_duration_seconds{status="pending"}) > 60 * 60
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  - alert: TiDBClusterLogBackupStatusStopping
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} log backup in stopping status more than 1h.
    expr: |
      (dataflow_log_backup_execution_status_duration_seconds{status="stopping"}) > 60 * 60
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  - alert: TiDBClusterLogBackupLagHigh
    annotations:
      message: |
        Tenant/Cluster: {{ $labels.tenant }} / {{ $labels.cluster_id }}
        LogBackup: {{ $labels.backup_name }} / {{ $labels.backup_id }}
        BackupLag: {{ printf "%.0f" $value }}s
        BackupCreatedAt: {{ $labels.create_at }}
        SOP: https://pingcap.feishu.cn/wiki/VTLewR7Ayi0aoZkNfJdcNoYjnUh
    expr: |
      ((dataflow_log_backup_lag_duration_seconds) >= 30 * 60)
      and
      ((dataflow_log_backup_lag_duration_seconds) < 12 * 3600)
    for: 15m
    labels:
      severity: major
      response_class: ticket
      component: br
      oncall_team: tidb_x_tools
  # Exclude clusters with frequent resume/pause actions that cause fake alarms.
  - alert: TiDBClusterLogBackupLagTooHigh
    annotations:
      message: |
        Tenant/Cluster: {{ $labels.tenant }} / {{ $labels.cluster_id }}
        LogBackup: {{ $labels.backup_name }} / {{ $labels.backup_id }}
        BackupLag: {{ printf "%.0f" $value }}s
        BackupCreatedAt: {{ $labels.create_at }}
        SOP: https://pingcap.feishu.cn/wiki/VTLewR7Ayi0aoZkNfJdcNoYjnUh
    expr: |
      (dataflow_log_backup_lag_duration_seconds{cluster_id!~"^(10076823780313608906|10662888935658790607|10070106825815339119)$"}) >= 12 * 3600
    for: 30m
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  # log backup truncate running
  - alert: TiDBClusterLogBackupTruncateDurationMoreThan6h
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} log backup {{ $labels.backup_id }} truncate running duration more than 6h.
    expr: |
      (dataflow_log_backup_truncate_status_duration_seconds{status="running"}) > 6 * 60 * 60
    labels:
      severity: major
      response_class: ticket
      component: br
      oncall_team: tidb_x_tools
  - alert: TiDBClusterLogBackupTruncateDurationMoreThan24h
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} log backup {{ $labels.backup_id }} truncate running duration more than 24h.
    expr: |
      (dataflow_log_backup_truncate_status_duration_seconds{status="running"}) > 24 * 60 * 60
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  # TiDB Cluster restore alerts, make it 60 minutes since cluster creation will take 30~40 minutes
  - alert: TiDBClusterRestoreStatusPending
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} restore in pending status more than 60 minutes.
    expr: |
      dataflow_restore_execution_status_duration_seconds{status="pending"} > 60 * 60
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  - alert: TiDBClusterRestoreStatusFailed
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} restore status is failed.
    expr: |
      dataflow_restore_execution_status_duration_seconds{status="failed"} > 0
    labels:
      severity: critical
      response_class: page
      alerttype: event
      component: br
      oncall_team: tidb_x_tools
  - alert: TiDBClusterRestoreDurationCritical
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} restore running duration more than 20 hours
    expr: |
      dataflow_restore_execution_status_duration_seconds{status!~"failed|success|finished|deleted"} > 60 * 60 * 20
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools

  # backup export alerts
  - alert: BackupExportDurationTooLong
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} backup export running duration more than 30m.
    expr: |
      dataflow_backup_export_status{status=~"running|pending"} == 1
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: br
      oncall_team: tidb_x_tools
  - alert: BackupExportStatusFailed
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_id }}/{{ $labels.cluster_id }} {{ $labels.backup_type }} backup export status is failed.
    expr: |
      dataflow_backup_export_status{status="failed"} == 1
    for: 0m
    labels:
      severity: major
      response_class: ticket
      component: br
      oncall_team: tidb_x_tools