groups:
- name: backup-restore
  rules:
  # PiTR alerts
  - alert: LogBackupRunningRPOMoreThan30m
    annotations:
      summary: RPO of log backup is too high
      message: The log backup task {{ $labels.task }} RPO of TiDB cluster {{ $labels.tenant }}/{{ $labels.tidb_cluster }} is more than {{ $value }}m
    expr: |
      max(time() - tidb_log_backup_last_checkpoint{tidb_cluster_id!~"^(1379661944638124241)$"} / 262144000) by (tenant, tidb_cluster, task) / 60 > 30
      and max(tidb_log_backup_last_checkpoint{tidb_cluster_id!~"^(1379661944638124241)$"}) by (tenant, tidb_cluster, task) > 0
      and max(tikv_log_backup_task_status{tidb_cluster_id!~"^(1379661944638124241)$"}) by (tenant, tidb_cluster, task) == 0
    for: 10m # reduce noise for pause/resume cluster
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  - alert: LogBackupRunningCrossRegionRPOMoreThan10m
    annotations:
      summary: Cross Region RPO of log backup is too high
      message: The log backup task {{ $labels.task }} cross region RPO of TiDB cluster {{ $labels.tenant }}/{{ $labels.tidb_cluster }} is more than {{ $value }}m
    expr: |
      max(time() - tidb_br_crr_safe_checkpoint{tidb_cluster_id!~"^(1379661944638124241)$"} / 262144000) by (tenant, tidb_cluster_id, task) / 60 > 10
      and max(tidb_br_crr_safe_checkpoint{tidb_cluster_id!~"^(1379661944638124241)$"}) by (tenant, tidb_cluster_id, task) > 0
      and max(tikv_log_backup_task_status{tidb_cluster_id!~"^(1379661944638124241)$"}) by (tenant, tidb_cluster_id, task) == 0
    for: 3m
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  # Exclude clusters with frequent resume/pause actions that cause fake alarms.
  - alert: LogBackupPausingMoreThan12h
    annotations:
      summary: Log backup paused more than 12h
      message: The log backup task {{ $labels.task }} of TiDB cluster {{ $labels.tenant }}/{{ $labels.tidb_cluster }} paused {{ $value }}h
    expr: |
      max(time() - tidb_log_backup_last_checkpoint{tidb_cluster_id!~"^(10076823780313608906|10662888935658790607|10070106825815339119)$"} / 262144000) by (tenant, tidb_cluster, task) / 3600 > 12
      and max(tidb_log_backup_last_checkpoint{tidb_cluster_id!~"^(10076823780313608906|10662888935658790607|10070106825815339119)$"}) by (tenant, tidb_cluster, task) > 0
      and max(tikv_log_backup_task_status{tidb_cluster_id!~"^(10076823780313608906|10662888935658790607|10070106825815339119)$"}) by (tenant, tidb_cluster, task) == 1
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  - alert: LogBackupFailed
    annotations:
      summary: Log backup failed
      message: The log backup task {{ $labels.task }} of TiDB cluster {{ $labels.tenant }}/{{ $labels.tidb_cluster }} failed
    expr: |
      max(tikv_log_backup_task_status) by (tenant, tidb_cluster, task) == 2
      and max(tidb_log_backup_last_checkpoint) by (tenant, tidb_cluster, task) > 0
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  - alert: LogBackupGCSafePointExceedsCheckpoint
    annotations:
      summary: Log backup gc safe point excceeds checkpoint
      message: The log backup of TiDB cluster {{ $labels.tenant }}/{{ $labels.tidb_cluster }} gc safe point ts exceeds checkpoint ts
    expr: |
      min(tidb_log_backup_last_checkpoint) by (tenant, tidb_cluster, instance) - max(tikv_gcworker_autogc_safe_point) by (tenant, tidb_cluster, instance) < 0
    labels:
      severity: critical
      response_class: page
      component: br
      oncall_team: tidb_x_tools
  - alert: BackupPodUnhealthy
    annotations:
      summary: Backup pod status unhealthy
      message: The backup pod {{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.pod }} is {{ $labels.phase }} for 5 mins.
    expr: |
      avg by (cluster, namespace, pod, phase) (kube_pod_status_phase{pod=~"backup-ab.*", phase=~"Pending|Unknown|Failed"}) > 0
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: br
      oncall_team: tidb_x_tools
  - alert: RestorePodUnhealthy
    annotations:
      summary: Backup pod status unhealthy
      message: The restore pod {{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.pod }} is {{ $labels.phase }} for 5 mins.
    expr: |
      avg by (cluster, namespace, pod, phase) (kube_pod_status_phase{pod=~"restore.*", phase=~"Pending|Unknown|Failed"}) > 0
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: br
      oncall_team: tidb_x_tools
  - alert: LogBackupPodUnhealthy
    annotations:
      summary: Backup pod status unhealthy
      message: The log backup pod {{ $labels.cluster }}/{{ $labels.namespace }}/{{ $labels.pod }} is {{ $labels.phase }} for 5 mins.
    expr: |
      avg by (cluster, namespace, pod, phase) (kube_pod_status_phase{pod=~"backup-log-ab.*", phase=~"Pending|Unknown|Failed"}) > 0
    for: 5m
    labels:
      severity: major
      response_class: ticket
      component: br
      oncall_team: tidb_x_tools