groups:
  - name: scale_status.rules
    interval: 30s
    rules:
      - alert: TiKVOrTiFlashScaleInStuck
        expr: |
          (
            (
              infra_exporter_tikv_or_tiflash_leader_count{component=~"tikv|tiflash", scale_type="scaleIn"} > 0
              OR
              infra_exporter_tikv_or_tiflash_region_count{component=~"tikv|tiflash", scale_type="scaleIn"} > 0
            )
            AND
            (
              infra_exporter_tikv_or_tiflash_leader_count{component=~"tikv|tiflash", scale_type="scaleIn"} == 0
              OR
              (
                changes(infra_exporter_tikv_or_tiflash_leader_count{component=~"tikv|tiflash", scale_type="scaleIn"}[10m]) == 0
                AND
                count_over_time(infra_exporter_tikv_or_tiflash_leader_count{component=~"tikv|tiflash", scale_type="scaleIn"}[10m]) >= 5
                AND
                (time() - timestamp(infra_exporter_tikv_or_tiflash_leader_count{component=~"tikv|tiflash", scale_type="scaleIn"})) < 5 * 60 # 5 minutes
              )
            )
            AND
            (
              infra_exporter_tikv_or_tiflash_region_count{component=~"tikv|tiflash", scale_type="scaleIn"} == 0
              OR
              (
                changes(infra_exporter_tikv_or_tiflash_region_count{component=~"tikv|tiflash", scale_type="scaleIn"}[10m]) == 0
                AND
                count_over_time(infra_exporter_tikv_or_tiflash_region_count{component=~"tikv|tiflash", scale_type="scaleIn"}[10m]) >= 5
                AND
                (time() - timestamp(infra_exporter_tikv_or_tiflash_region_count{component=~"tikv|tiflash", scale_type="scaleIn"})) < 5 * 60 # 5 minutes
              )
            )
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          message: "TiKV/TiFlash member {{ $labels.name }} ({{ $labels.component }}) scale in stuck: no leader/region progress for 10+ minutes. Cluster: {{ $labels.cluster_id }}, Group: {{ $labels.group_id }}"

      - alert: TiKVOrTiFlashScaleUpDownStuck
        expr: |
          infra_exporter_tikv_or_tiflash_leader_count{component=~"tikv|tiflash", scale_type=~"scaleUp|scaleDown"} > 0
          AND
          changes(infra_exporter_tikv_or_tiflash_leader_count{component=~"tikv|tiflash", scale_type=~"scaleUp|scaleDown"}[10m]) == 0
          AND
          count_over_time(infra_exporter_tikv_or_tiflash_leader_count{component=~"tikv|tiflash", scale_type=~"scaleUp|scaleDown"}[10m]) >= 5
          AND
          (time() - timestamp(infra_exporter_tikv_or_tiflash_leader_count{component=~"tikv|tiflash", scale_type=~"scaleUp|scaleDown"})) < 5 * 60 # 5 minutes
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          message: "TiKV/TiFlash member {{ $labels.name }} ({{ $labels.component }}) scale up/down stuck: no leader progress for 10+ minutes. Cluster: {{ $labels.cluster_id }}, Group: {{ $labels.group_id }}"


      - alert: ComponentScaleTimeout
        expr: |
          infra_exporter_component_status{component!~"tikv|tiflash", scale_type!="none", component_status="Scaling"} > 0
          AND
          (time() - timestamp(infra_exporter_component_status{component!~"tikv|tiflash", scale_type!="none", component_status="Scaling"})) < 5 * 60
        for: 100m
        labels:
          severity: major
          response_class: ticket
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          message: "Component {{ $labels.component }} scale timeout: scale_type={{ $labels.scale_type }} for 100+ minutes. Cluster: {{ $labels.cluster_id }}, Group: {{ $labels.group_id }}"

      - alert: TiKVOrTiFlashComponentScaleOutOrStorageSizeChangeTimeout
        expr: |
          infra_exporter_component_status{component=~"tikv|tiflash", scale_type=~"scaleOut|storageSizeChange", component_status="Scaling"} > 0
          AND
          (time() - timestamp(infra_exporter_component_status{component=~"tikv|tiflash", scale_type=~"scaleOut|storageSizeChange", component_status="Scaling"})) < 5 * 60
        for: 90m
        labels:
          severity: critical
          response_class: page
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          message: "TiKV/TiFlash component {{ $labels.component }} scale out or storage size change timeout: scale_type={{ $labels.scale_type }} for 90+ minutes. Cluster: {{ $labels.cluster_id }}, Group: {{ $labels.group_id }}"

      - alert: TiKVOrTiFlashMemberScaleOutOrStorageSizeChangeTimeout
        expr: |
          infra_exporter_component_member_status{component=~"tikv|tiflash", scale_type=~"scaleOut|storageSizeChange", component_status="Scaling"} > 0
          AND
          (time() - timestamp(infra_exporter_component_member_status{component=~"tikv|tiflash", scale_type=~"scaleOut|storageSizeChange", component_status="Scaling"})) < 5 * 60 # 5 minutes
        for: 35m
        labels:
          severity: major
          response_class: ticket
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          message: "TiKV/TiFlash member {{ $labels.name }} ({{ $labels.component }}) scale out or storage size change timeout: scale_type={{ $labels.scale_type }} for 35+ minutes. Cluster: {{ $labels.cluster_id }}, Group: {{ $labels.group_id }}"


      - alert: TiKVOrTiFlashHang
        expr: |
          (time() - infra_exporter_tikv_or_tiflash_pod_create_time{component=~"tikv|tiflash", scale_type=~"scaleUp|scaleOut"}) > 15 * 60
          AND
          (
            (
              infra_exporter_tikv_or_tiflash_pod_create_time{component=~"tikv|tiflash", scale_type=~"scaleUp|scaleOut"}
              > on(cluster_id, tenant_id, project_id, namespace, group_id, component, scale_type, name) group_left()
              infra_exporter_tikv_or_tiflash_start_timestamp
            )
            OR
            (
              infra_exporter_tikv_or_tiflash_pod_create_time{component=~"tikv|tiflash", scale_type=~"scaleUp|scaleOut"}
              unless on(cluster_id, tenant_id, project_id, namespace, group_id, component, scale_type, name)
              infra_exporter_tikv_or_tiflash_start_timestamp
            )
          )
        for: 2m
        labels:
          severity: major
          response_class: ticket
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          message: "Member {{ $labels.name }} ({{ $labels.component }}) pod created but store not started for 15+ minutes. Cluster: {{ $labels.cluster_id }}, Group: {{ $labels.group_id }}"


      - alert: TiDBCloudSyncDuringTooLong
        expr: |
          infra_exporter_last_update_time_from_tidbcloud{cluster_full_synced="false"} > 0
          AND
          changes(infra_exporter_last_update_time_from_tidbcloud{cluster_full_synced="false"}[15m]) == 0
          AND
          count_over_time(infra_exporter_last_update_time_from_tidbcloud{cluster_full_synced="false"}[15m]) >= 8
          AND
          (time() - timestamp(infra_exporter_last_update_time_from_tidbcloud{cluster_full_synced="false"})) < 5 * 60
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          message: "TiDBCloud sync duration too long: last_update_time_from_tidbcloud not updated for 15+ minutes while cluster_full_synced=false. Cluster: {{ $labels.cluster_id }}"



      # - alert: ClusterReconcileErrorTooManyIn15m
      #   expr: |
      #     increase(infra_cluster_reconcile_error[15m]) > 20
      #     AND
      #     count_over_time(infra_cluster_reconcile_error[15m]) >= 8
      #   for: 1m
      #   labels:
      #     severity: major
      #     component: infra-provider
      #   annotations:
      #     message: "Cluster reconcile errors too many: {{ $value }} errors in 15m for {{ $labels.namespace }}/{{ $labels.name }} (threshold: >20)."



      # - alert: ScaleStatusTrackedNotSyncedFor15m
      #   expr: |
      #     infra_exporter_scale_status_tracked{is_synced="false"} > 0
      #     AND
      #     changes(infra_exporter_scale_status_tracked{is_synced="false"}[15m]) == 0
      #     AND
      #     count_over_time(infra_exporter_scale_status_tracked{is_synced="false"}[15m]) >= 8
      #     AND
      #     (time() - timestamp(infra_exporter_scale_status_tracked{is_synced="false"})) < 5 * 60
      #   for: 5m
      #   labels:
      #     severity: major
      #     component: infra-provider
      #   annotations:
      #     message: "Scale status tracked not synced: is_synced=false and not updated for 15+ minutes. Cluster: {{ $labels.cluster_id }}"



      - alert: NodePoolCreateNodesTooLong
        expr: |
          infra_exporter_component_status{instance_type_scale_phase="CreatingNodes", component_status="Normal"} > 0
          AND
          (time() - timestamp(infra_exporter_component_status{instance_type_scale_phase="CreatingNodes", component_status="Normal"})) < 5 * 60
        for: 30m
        labels:
          severity: major
          response_class: ticket
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          message: "Node pool create duration too long: component {{ $labels.component }} stayed Normal during instance_type_scale_phase=CreatingNodes for 30+ minutes. Cluster: {{ $labels.cluster_id }}"

      - alert: NodePoolDeleteNodesTooLong
        expr: |
          infra_exporter_component_status{instance_type_scale_phase="DeletingNodes", component_status="Normal"} > 0
          AND
          (time() - timestamp(infra_exporter_component_status{instance_type_scale_phase="DeletingNodes", component_status="Normal"})) < 5 * 60
        for: 30m
        labels:
          severity: major
          response_class: ticket
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          message: "Node pool delete duration too long: component {{ $labels.component }} stayed Normal during instance_type_scale_phase=DeletingNodes for 30+ minutes. Cluster: {{ $labels.cluster_id }}"

      - alert: ComponentMemberScaleDurationTooLong
        expr: |
          infra_exporter_component_member_status{component!~"tikv|tiflash", scale_type!="none", component_status="Scaling"} > 0
          AND
          (time() - timestamp(infra_exporter_component_member_status{component!~"tikv|tiflash", scale_type!="none", component_status="Scaling"})) < 5 * 60
        for: 30m
        labels:
          severity: major
          response_class: ticket
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          message: "Component member {{ $labels.name }} ({{ $labels.component }}) scale duration too long: scale_type={{ $labels.scale_type }} for 30+ minutes. Cluster: {{ $labels.cluster_id }}, Group: {{ $labels.group_id }}"
      - alert: ScaleProtectionMissingOnNode
        expr: |
          infra_cluster_scale_protection_missing_node{
            status!~"Pending|Creating|Deleting|Suspend"
          } == 1
        labels:
          severity: major
          response_class: ticket
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          summary: "Node missing scale-in protection ({{ $labels.name }}/{{ $labels.namespace }})"
          message: |
            Cluster {{ $labels.name }}/{{ $labels.namespace }} has a TiDB data node missing scale-in protection.
            Node={{ $labels.node }}, instanceID={{ $labels.instance_id }}, component={{ $labels.component }}.
            node_annotation_missing={{ $labels.node_annotation_missing }}, aws_missing={{ $labels.aws_missing }}.
      
      - alert: ScaleProtectionCheckStuckFailed
        expr: |
          increase(infra_cluster_scale_protection_check_skipped[12h]) >= 3
        labels:
          severity: major
          response_class: ticket
          component: infra-provider
          oncall_team: dedicated_service
        annotations:
          summary: "Scale protection checker repeatedly failing for cluster {{ $labels.name }}/{{ $labels.namespace }}"
          message: |
            Scale protection check for cluster {{ $labels.name }}/{{ $labels.namespace }} has failed at least 3 times in the last 12 hours.
            reason={{ $labels.reason }}, node={{ $labels.node }}.
            The controller cannot reliably verify scale-in protection state; inspect infra_cluster_scale_protection_check_skipped and controller logs for details.