groups:
- name: infra-exporter
  rules:
  - alert: ClusterPDStatusIsScalingMoreThan1h
    annotations:
      message: |
        The pd status of the cluster {{ $labels.exported_namespace }}/{{ $labels.name }} is scaling for more than 1h.
    expr: infra_exporter_cluster{pd_status="Scaling"} == 1
    for: 60m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: ClusterPDStatusIsUpgradingMoreThan1h
    annotations:
      message: |
        The pd status of the cluster {{ $labels.exported_namespace }}/{{ $labels.name }} is scaling for more than 1h.
    expr: infra_exporter_cluster{pd_status="Upgrading"} == 1
    for: 60m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: ClusterCRCreationDurationTooLong
    annotations:
      message: |
        The Cluster CR {{ $labels.exported_namespace }}/{{ $labels.name }} creation time exceeds 25 minutes.
    expr: infra_exporter_cluster{status=~"^$|Creating"} == 1
    for: 40m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: ClusterCRScalingDurationTooLong
    annotations:
      message: |
        Tenant {{ $labels.tenant_id }} 's Cluster CR {{ $labels.exported_namespace }}/{{ $labels.name }} scaling time exceeds 40 minutes.
        Reason: {{ $labels.reason }}
    expr: infra_exporter_cluster{status="Scaling"} == 1
    for: 40m
    labels:
      severity: major
      response_class: ticket
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: ClusterCRKeepDeletingTooLong
    annotations:
      message: |
        The Cluster CR {{ $labels.exported_namespace }}/{{ $labels.name }} keeps being deleting time exceeds 30 minutes.
    expr: infra_exporter_cluster{status="Deleting"} == 1
    for: 30m
    labels:
      severity: major
      response_class: ticket
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: ClusterCRKeepPausedTooLong
    annotations:
      message: |
        The Cluster CR {{ $labels.exported_namespace }}/{{ $labels.name }} keeps being paused time exceeds 60 minutes.
    expr: infra_exporter_cluster{paused="true"} == 1
    for: 60m
    labels:
      severity: major
      response_class: ticket
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: ClusterCRKeepPausedMoreThan2Hour
    annotations:
      message: |
        The Cluster CR {{ $labels.exported_namespace }}/{{ $labels.name }} keeps being paused time exceeds 120 minutes. Do not forget to unpause it.
    expr: infra_exporter_cluster{paused="true"} == 1
    for: 2h
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: PrivateLinkServiceCreationDurationTooLong
    annotations:
      message: |
        The PirvateLink Service {{ $labels.exported_namespace }}/{{ $labels.name }} creation time exceeds 15 minutes.
    expr: infra_exporter_cluster{privatelinkserivce_status=~"^$|Creating"} == 1
    for: 15m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: PrivateLinkServiceKeepDeletingTooLong
    annotations:
      message: |
        The PirvateLink Service {{ $labels.exported_namespace }}/{{ $labels.name }} keeps being deleting time exceeds 15 minutes.
    expr: infra_exporter_cluster{privatelinkserivce_status="Deleting"} == 1
    for: 15m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  # Regional UpgradeOperation
  - alert: RegionalUpgradeOperationMissTheSchedule
    annotations:
      message: |
        The regional UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} does not start at the scheduled time.
    expr: regional_server_upgradeoperation_info{phase="Pending", miss_schedule="true",paused="false"} == 1
    for: 1m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: RegionalUpgradeOperationTimeout
    annotations:
      message: |
        The regional UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} exceeds the desired duration.
    expr: regional_server_upgradeoperation_info{phase="Running", timeout="true",paused="false"} == 1
    for: 1m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: RegionalUpgradeOperationFailed
    annotations:
      message: |
        The regional UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} failed.
    expr: regional_server_upgradeoperation_info{phase="Failed",paused="false"} == 1
    for: 1m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: RegionalUpgradeControlPlaneTimeout
    annotations:
      message: |
        The regional UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades control plane for over 32 minutes.
    expr: regional_server_upgradeoperation_info{phase="Running", current_target="controlplane", paused="false"} == 1
    for: 32m
    labels:
      severity: critical
      response_class: page
      component: dedicated-regional-server
      oncall_team: dedicated_service
  - alert: RegionalUpgradeAdminNodesTimeout
    annotations:
      message: |
        The regional UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades admin nodes for over 30 minutes.
    expr: regional_server_upgradeoperation_info{phase="Running", current_target="admin-nodes", paused="false"} == 1
    for: 30m
    labels:
      severity: major
      response_class: ticket
      component: dedicated-regional-server
      oncall_team: dedicated_service
  - alert: RegionalUpgradeEKSAddonsTimeout
    annotations:
      message: |
        The regional UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades eks add-ons for over 45 minutes.
    expr: regional_server_upgradeoperation_info{phase="Running", current_target="addons", paused="false"} == 1
    for: 45m
    labels:
      severity: critical
      response_class: page
      component: dedicated-regional-server
      oncall_team: dedicated_service
  - alert: RegionalUpgradeCDCNodesTimeout
    annotations:
      message: |
        The regional UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades cdc for over 20 minutes.
    expr: regional_server_upgradeoperation_info{phase="Running", current_target="cdc-nodes", paused="false"} == 1
    for: 20m
    labels:
      severity: critical
      response_class: page
      component: dedicated-regional-server
      oncall_team: dedicated_service
  - alert: RegionalUpgradeDMNodesTimeout
    annotations:
      message: |
        The regional UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades dm for over 20 minutes.
    expr: regional_server_upgradeoperation_info{phase="Running", current_target="dm-nodes", paused="false"} == 1
    for: 20m
    labels:
      severity: critical
      response_class: page
      component: dedicated-regional-server
      oncall_team: dedicated_service
  - alert: RegionalUpgradeO11yK8sNodesTimeout
    annotations:
      message: |
        The regional UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades o11y-k8s for over 20 minutes.
    expr: regional_server_upgradeoperation_info{phase="Running", current_target="o11y-k8s-nodes", paused="false"} == 1
    for: 20m
    labels:
      severity: critical
      response_class: page
      component: dedicated-regional-server
      oncall_team: dedicated_service
  - alert: RegionalUpgradeO11yTiDBNodesTimeout
    annotations:
      message: |
        The regional UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades o11y-tidb for over 20 minutes.
    expr: regional_server_upgradeoperation_info{phase="Running", current_target="o11y-tidb-nodes", paused="false"} == 1
    for: 20m
    labels:
      severity: critical
      response_class: page
      component: dedicated-regional-server
      oncall_team: dedicated_service
  # Cluster UpgradeOperation
  - alert: UpgradeOperationMissTheSchedule
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} does not start at the scheduled time.
    expr: infra_exporter_upgradeoperation{phase="Pending", miss_schedule="true",paused="false"} == 1
    for: 1m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeOperationTimeout
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} exceeds the desired duration.
    expr: infra_exporter_upgradeoperation{phase="Running", timeout="true",paused="false"} == 1
    for: 1m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeOperationFailed
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} failed.
    expr: infra_exporter_upgradeoperation{phase="Failed",paused="false"} == 1
    for: 1m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeEKSControlPlaneTimeout
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades eks control plane for over 15 minutes.
    expr: infra_exporter_upgradeoperation{phase="Running", current_target="controlplane",paused="false"} == 1
    for: 15m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeGKEControlPlaneTimeout
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades gke control plane for over 32 minutes.
    expr: infra_exporter_upgradeoperation{phase="Running", current_target="controlplane",paused="false"} == 1
    for: 32m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeEKSAdminNodesTimeout
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades eks admin nodes for over 30 minutes.
    expr: infra_exporter_upgradeoperation{phase="Running", current_target="admin-nodes",paused="false"} == 1
    for: 30m
    labels:
      severity: major
      response_class: ticket
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeTiDBNodesTimeout
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades tidb nodes for over one hour.
    expr: infra_exporter_upgradeoperation{phase="Running", current_target="tidb-nodes",paused="false"} == 1
    for: 1h
    labels:
      severity: major
      response_class: ticket
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra