groups:
- name: infra-exporter
  rules:
  - alert: ClusterPDStatusIsScalingMoreThan1h
    annotations:
      message: |
        The pd status of the cluster {{ $labels.exported_namespace }}/{{ $labels.name }} is scaling for more than 1h.
    expr: infra_exporter_cluster{pd_status="Scaling"} == 1
    for: 60m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: ClusterPDStatusIsUpgradingMoreThan1h
    annotations:
      message: |
        The pd status of the cluster {{ $labels.exported_namespace }}/{{ $labels.name }} is scaling for more than 1h.
    expr: infra_exporter_cluster{pd_status="Upgrading"} == 1
    for: 60m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: NetworkCRCreationDurationTooLong
    annotations:
      message: |
        The Network CR {{ $labels.exported_namespace }}/{{ $labels.name }} creation time exceeds 30 minutes.
    expr: infra_exporter_network{status=~"^$|Creating|WakingUp", exported_provider!~".*free-tier"} == 1
    for: 30m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: ClusterCRCreationDurationTooLong
    annotations:
      message: |
        The Cluster CR {{ $labels.exported_namespace }}/{{ $labels.name }} creation time exceeds 25 minutes.
    expr: infra_exporter_cluster{status=~"^$|Creating", network_name!~".*free-tier.*", tenant_id!="1372813089192471291"} == 1
    for: 40m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: NetworkKeepHibernatingTooLong
    annotations:
      message: |
        The Network CR {{ $labels.exported_namespace }}/{{ $labels.name }} keeps hibernating exceeds 40 minutes.
    expr: infra_exporter_network{status="Hibernating", exported_provider!~".*free-tier"} == 1
    for: 40m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: NetworkCRKeepDeletingTooLong
    annotations:
      message: |
        The Network CR {{ $labels.exported_namespace }}/{{ $labels.name }} keeps being deleting exceeds 30 minutes.
    expr: infra_exporter_network{status=~"Deleting|PendingDelete", exported_provider!~".*free-tier", tenant_id!="1372813089187791289"} == 1 # mute QA's networks
    for: 30m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: QANetworkCRKeepDeletingTooLong
    annotations:
      message: |
        The Network CR {{ $labels.exported_namespace }}/{{ $labels.name }} keeps being deleting exceeds 30 minutes.
    expr: infra_exporter_network{status=~"Deleting|PendingDelete", exported_provider!~".*free-tier", tenant_id="1372813089187791289"} == 1
    for: 30m
    labels:
      severity: major
      response_class: ticket
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: ClusterCRKeepDeletingTooLong
    annotations:
      message: |
        The Cluster CR {{ $labels.exported_namespace }}/{{ $labels.name }} keeps being deleting time exceeds 30 minutes.
    expr: infra_exporter_cluster{status="Deleting", network_name!~".*free-tier.*"} == 1
    for: 30m
    labels:
      severity: major
      response_class: ticket
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: PrivateLinkServiceCreationDurationTooLong
    annotations:
      message: |
        The PirvateLink Service {{ $labels.exported_namespace }}/{{ $labels.name }} creation time exceeds 15 minutes.
    expr: infra_exporter_cluster{privatelinkserivce_status=~"^$|Creating", network_name!~".*free-tier.*"} == 1
    for: 15m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: PrivateLinkServiceKeepDeletingTooLong
    annotations:
      message: |
        The PirvateLink Service {{ $labels.exported_namespace }}/{{ $labels.name }} keeps being deleting time exceeds 15 minutes.
    expr: infra_exporter_cluster{privatelinkserivce_status="Deleting", network_name!~".*free-tier.*"} == 1
    for: 15m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeOperationMissTheSchedule
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} does not start at the scheduled time.
    expr: infra_exporter_upgradeoperation{phase="Pending", miss_schedule="true",paused="false"} == 1
    for: 1m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeOperationTimeout
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} exceeds the desired duration.
    expr: infra_exporter_upgradeoperation{phase="Running", timeout="true",paused="false"} == 1
    for: 1m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeOperationFailed
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} failed.
    expr: infra_exporter_upgradeoperation{phase="Failed",paused="false"} == 1
    for: 1m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeEKSControlPlaneTimeout
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades eks control plane for over 15 minutes.
    expr: infra_exporter_upgradeoperation{phase="Running", current_target="controlplane", network_name=~"eks.*",paused="false"} == 1
    for: 15m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeGKEControlPlaneTimeout
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades gke control plane for over 32 minutes.
    expr: infra_exporter_upgradeoperation{phase="Running", current_target="controlplane", network_name=~"gke.*",paused="false"} == 1
    for: 32m
    labels:
      severity: critical
      response_class: page
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeEKSAdminNodesTimeout
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades eks admin nodes for over 30 minutes.
    expr: infra_exporter_upgradeoperation{phase="Running", current_target="admin-nodes", network_name=~"eks.*",paused="false"} == 1
    for: 30m
    labels:
      severity: major
      response_class: ticket
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra
  - alert: UpgradeTiDBNodesTimeout
    annotations:
      message: |
        The UpgradeOperation {{ $labels.exported_namespace }}/{{ $labels.name }} upgrades tidb nodes for over one hour.
    expr: infra_exporter_upgradeoperation{phase="Running", current_target="tidb-nodes",paused="false"} == 1
    for: 1h
    labels:
      severity: major
      response_class: ticket
      service: infra-provider
      component: infra-provider
      oncall_team: dedicated_service
      type: infra