groups:
- name: TiDBCloudIncident
  rules:
  - alert: IncidentInstanceCreationDurationMoreThan1h
    annotations:
      message: The TiDB cluster {{ $labels.tenant_name }}/{{ $labels.name }} creation time exceeds 60 minutes
    expr: |
      dbaas_tidb_cluster_wait_ready_duration_seconds{provider_type=~"dedicate", status!="recovering"} > 60 * 60
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
  - alert: IncidentClusterResumeDurationMoreThan20m
    annotations:
      message: The TiDB cluster {{ $labels.cluster_id }} in {{ $labels.tenant_id }} resume time exceeds 20 minutes
    expr: |
      dbaas_tidb_cluster_resume_duration_seconds{} > 20 * 60
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
  - alert: IncidentClusterPauseDurationMoreThan20m
    annotations:
      message: The TiDB cluster {{ $labels.cluster_id }} in {{ $labels.tenant_id }} pause time exceeds 20 minutes
    expr: |
      dbaas_tidb_cluster_pause_duration_seconds{} > 20 * 60
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
  - alert: IncidentClusterScaleUpDurationMoreThan60m
    annotations:
      message: The TiDB Cluster {{ $labels.name }} scale-up duration exceeds 60 minutes
    expr: |
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{compute_up_down="1"}) > 0
      and on(cluster_id)
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{scale_out_in="-1"}) == 0
    for: 60m
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
  - alert: IncidentClusterScaleUpDurationMoreThan90m
    annotations:
      message: The TiDB Cluster {{ $labels.name }} scale-up duration exceeds 90 minutes
    expr: |
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{compute_up_down="1"}) > 0
      and on(cluster_id)
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{scale_out_in="-1"}) == 0
    for: 90m
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
  - alert: IncidentClusterScaleDownDurationMoreThan60m
    annotations:
      message: The TiDB Cluster {{ $labels.name }} scale-down duration exceeds 60 minutes
    expr: |
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{compute_up_down="-1"}) > 0
      and on(cluster_id)
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{scale_out_in="-1"}) == 0
    for: 60m
    labels:
      severity: major
      response_class: ticket
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: IncidentClusterScaleDownDurationMoreThan90m
    annotations:
      message: The TiDB Cluster {{ $labels.name }} scale-down duration exceeds 90 minutes
    expr: |
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{compute_up_down="-1"}) > 0
      and on(cluster_id)
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{scale_out_in="-1"}) == 0
    for: 90m
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
  - alert: IncidentClusterScaleOutDurationMoreThan60m
    annotations:
      message: The TiDB Cluster {{ $labels.name }} scale-out duration exceeds 60 minutes
    expr: |
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{scale_out_in="1"}) > 0
      and on(cluster_id)
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{scale_out_in="-1"}) == 0
    for: 60m
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
  - alert: IncidentClusterScaleOutDurationMoreThan90m
    annotations:
      message: The TiDB Cluster {{ $labels.name }} scale-out duration exceeds 90 minutes
    expr: |
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{scale_out_in="1"}) > 0
      and on(cluster_id)
      count by (tenant_id,cluster_id,name) (dbaas_tidb_cluster_scaling_info{scale_out_in="-1"}) == 0
    for: 90m
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
  - alert: IncidentClusterBackupTooManyFailures
    annotations:
      message: The number of failed backups in the last 48h exceeds 2.
    expr: |
      count(group by(backup_id) (increase(dataflow_backup_execution_status_duration_seconds{status="failed"}[48h]))) >= 2
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
  - alert: IncidentClusterRestoreTooManyFailures
    annotations:
      message: The number of failed restores in the last 48h exceeds 2.
    expr: |
      count(group by(restore_id) (increase(dataflow_restore_execution_status_duration_seconds{status="failed"}[48h]))) >= 2
    labels:
      severity: major
      response_class: ticket
      component: resilience
      oncall_team: resilience
  - alert: ControlPlaneServerlessRegionalAlertStorm
    annotations:
      message: The number of firing serverless alerts in {{ $labels.o11y_region }} exceeds 1000.
    expr: |
      max_over_time(
        count by (o11y_region) (
          ALERTS{
            alertstate="firing",
            o11y_region!="",
            job=~"(serverless.*)",
            alertname!="ControlPlaneServerlessRegionalAlertStorm"
          }
        )[5m:]
      ) > 1000
    labels:
      severity: critical
      response_class: page
      component: resilience
      oncall_team: resilience