groups:
  - name: serverless-service-br
    rules:
      - alert: BackupSkipped
        expr: sum(increase(serverless_backup_skip_number{namespace=~".*-ms"}[5m])) by (exported_provider,exported_region,kubernetes,schedule_time) > 0
        labels:
          tier: TiDB Serverless
          severity: warning
          component: br
          oncall_team: tidb_x_tools
          provider_type: aws-free-tier
        annotations:
          # provider and region conflicts with server-side labels, see prometheus honor_labels config
          message: Serverless {{ $labels.exported_provider }}/{{ $labels.exported_region }}/{{ $labels.kubernetes }} skipped {{ $labels.schedule_time }} backup as it is outdated.
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/serverless-service/serverless-service-br.rules.yaml

      - alert: RestoreRollbackFailed
        expr: sum(increase(serverless_restore_recover_failed_number{namespace=~".*-ms"}[5m])) by (cluster_id,type) > 0
        labels:
          tier: TiDB Serverless
          severity: warning
          component: br
          oncall_team: tidb_x_tools
          provider_type: aws-free-tier
        annotations:
          message: Serverless cluster {{ $labels.cluster_id }} rollback restore failed. {{ $labels.type }}
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/serverless-service/serverless-service-br.rules.yaml

      - alert: RestoreRunningTooLong
        expr: |
          serverless_restore_running_duration_seconds{namespace=~".*-ms"} > 600
        labels:
          tier: TiDB Serverless
          severity: warning
          component: br
          oncall_team: tidb_x_tools
          provider_type: aws-free-tier
        annotations:
          message: Serverless cluster {{ $labels.cluster_id }} restore job {{ $labels.restore_id }} running more than 600s.
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/serverless-service/serverless-service-br.rules.yaml

      - alert: RestoreFailed
        expr: sum(increase(serverless_restore_failed_number{namespace=~".*-ms"}[5m])) by (cluster_id,restore_id) > 0
        labels:
          tier: TiDB Serverless
          severity: warning
          component: br
          oncall_team: tidb_x_tools
          provider_type: aws-free-tier
        annotations:
          message: Serverless cluster {{ $labels.cluster_id }} restore job {{ $labels.restore_id }} failed.
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/serverless-service/serverless-service-br.rules.yaml

      - alert: BackupCronJobFailed
        expr: sum(increase(serverless_backup_cron_job_failed_number{namespace=~".*-ms"}[5m])) by (execute_time) > 0
        labels:
          tier: TiDB Serverless
          severity: warning
          component: br
          oncall_team: tidb_x_tools
          provider_type: aws-free-tier
        annotations:
          message: Serverless backup cron job failed at {{ $labels.execute_time }}. It may affect the generation of user backup.
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/serverless-service/serverless-service-br.rules.yaml

      - alert: BackupHandleEventFailed
        expr: |
          increase(serverless_backup_event_handle_total{status!="success"}[5m]) > 0
        labels:
          tier: TiDB Serverless
          severity: warning
          component: br
          oncall_team: tidb_x_tools
          provider_type: aws-free-tier
        annotations:
          message: Serverless backup handle cluster {{ $labels.cluster_id }} {{ $labels.event_type }} failed. The event id is {{ $labels.event_id }}. The consumption of eventbus will retry for 5 times, please confirm the final result.
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/serverless-service/serverless-service-br.rules.yaml

      - alert: BackupWorkerFailed
        expr: sum(increase(serverless_worker_duration_seconds_count{status!="ok",name=~"backupRetrieveWorker|backupCleanWorker|restoreSyncWorker"}[5m])) by (name) > 0
        labels:
          tier: TiDB Serverless
          severity: warning
          component: br
          oncall_team: tidb_x_tools
          provider_type: aws-free-tier
        annotations:
          message: Serverless BR worker {{ $labels.name }} failed.
          runbook_url: https://github.com/tidbcloud/runbooks/blob/master/rules/serverless-service/serverless-service-br.rules.yaml