# use infra-layer metrics to alert such case
# TODO: add shared-tier alert
# TODO: use infra metrics to alert infra status
groups:
- name: dedicated-cluster-operations
  rules:
  # TiDB Cluster alerts
  - alert: InfraClusterCreationDurationTooLong
    annotations:
      message: The TiDB cluster {{ $labels.tenant_name }}/{{ $labels.name }} creation time exceeds 20 minutes
    expr: |
      dbaas_tidb_cluster_wait_ready_duration_seconds{provider_type=~"dedicate", first_cluster="false", status!="recovering"} > 20 * 60
    labels:
      severity: major
      response_class: ticket
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: InfraClusterCreationDurationTooLong
    annotations:
      message: The Tidb cluster {{ $labels.tenant_name }}/{{ $labels.name }} creation time exceeds 30 minutes
    expr: |
      dbaas_tidb_cluster_wait_ready_duration_seconds{provider_type=~"dedicate", first_cluster="false", status!="recovering"} > 30 * 60
    labels:
      severity: critical
      response_class: page
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: TheProjectsFirstInfraClusterCreationDurationTooLong
    annotations:
      message: The projects first TiDB cluster {{ $labels.tenant_name }}/{{ $labels.name }} creation time exceeds 35 minutes
    expr: |
      dbaas_tidb_cluster_wait_ready_duration_seconds{provider_type=~"dedicate", first_cluster="true", status!="recovering"} > 35 * 60
    labels:
      severity: major
      response_class: ticket
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: TheProjectsFirstInfraClusterCreationDurationTooLong
    annotations:
      message: The projects first TiDB cluster {{ $labels.tenant_name }}/{{ $labels.name }} creation time exceeds 50 minutes
    expr: |
      dbaas_tidb_cluster_wait_ready_duration_seconds{provider_type=~"dedicate", first_cluster="true", status!="recovering"} > 50 * 60
    labels:
      severity: critical
      response_class: page
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: InfraClusterPauseDurationTooLong
    annotations:
      message: The TiDB cluster {{ $labels.cluster_id }} in {{ $labels.tenant_id }} pause time exceeds 20 minutes
    expr: |
      dbaas_tidb_cluster_pause_duration_seconds{} > 20 * 60
    labels:
      severity: major
      response_class: ticket
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: InfraClusterPauseDurationTooLong
    annotations:
      message: The TiDB cluster {{ $labels.cluster_id }} in {{ $labels.tenant_id }} pause time exceeds 30 minutes
    expr: |
      dbaas_tidb_cluster_pause_duration_seconds{} > 30 * 60
    labels:
      severity: critical
      response_class: page
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: InfraClusterResumeDurationTooLong
    annotations:
      message: The TiDB cluster {{ $labels.cluster_id }} in {{ $labels.tenant_id }} resume time exceeds 20 minutes
    expr: |
      dbaas_tidb_cluster_resume_duration_seconds{} > 20 * 60
    labels:
      severity: major
      response_class: ticket
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: InfraClusterResumeDurationTooLong
    annotations:
      message: The TiDB cluster {{ $labels.cluster_id }} in {{ $labels.tenant_id }} resume time exceeds 30 minutes
    expr: |
      dbaas_tidb_cluster_resume_duration_seconds{} > 30 * 60
    labels:
      severity: critical
      response_class: page
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: NuroInfraClusterResumeStart # alert specific for Nuro's cluster 1379661944635814096
    annotations:
      message: Nuro's cluster {{ $labels.cluster_id }} resumes with high risk to fail
    expr: |
      dbaas_tidb_cluster_resume_duration_seconds{cluster_id="1379661944635814096"} > 10
    labels:
      severity: critical
      response_class: page
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: InfraClusterStatusUnavailable
    annotations:
      message: The TiDB cluster {{ $labels.tenant_name }}/{{ $labels.name }} status is unavailable for 15 mins.
        please check pd tikv tidb component status.
    expr: |
      dbaas_tidb_cluster_info{status="unavailable", tenant_plan!="free_trial"} == 1
    for: 15m
    labels:
      severity: critical
      response_class: page
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: InfraClusterScalingDurationMoreThan1h
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_name }}/{{ $labels.name }} scaling (not scale-in) duration more than 1h
    expr: |
      dbaas_tidb_cluster_scaling_info{scale_out_in!="-1"} == 1
    for: 60m
    labels:
      severity: critical
      response_class: page
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: InfraClusterUpgradingDurationMoreThan1h
    annotations:
      message: The TiDB Cluster {{ $labels.tenant_name }}/{{ $labels.name }} upgrading duration more than 1h
    expr: |
      dbaas_tidb_cluster_info{status="upgrading", tenant_plan!="free_trial"} == 1
    for: 60m
    labels:
      severity: critical
      response_class: page
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: InfraClusterPrivateLinkServiceCreationDurationTooLong
    annotations:
      message: The TiDB Cluster {{ $labels.exported_namespace }}/{{ $labels.name }} creating privatelink service more than 15 mins.
    expr: |
      sum(infra_api_resources_cluster_privatelink_service_creation_duration_seconds{}) without (pod,instance) > 15 * 60
    labels:
      severity: critical
      response_class: page
      component: tidb-cluster
      oncall_team: dedicated_service
  - alert: CrossplaneNotResponsive
    annotations:
      message: The crossplane resource {{ $labels.name }} keep not responsive more than 15 mins.
    expr: |
      workqueue_depth{namespace="infra", service="aws-provider-upstream-metric"}
      / on (name)
        sum by (name) (
        label_replace(
          rate(
            controller_runtime_reconcile_total{namespace="infra",service="aws-provider-upstream-metric"}[5m]
          ),
          "name",
          "$1",
          "controller",
          "(.*)"
        )
      ) > 30
    # TODO reduce this time
    for: 15m
    labels:
      severity: major
      response_class: ticket
      component: tidb-cluster
      oncall_team: dedicated_service