groups:
  - name: nextgen-autoscaler-engine
    interval: 1m
    rules:
      - alert: TiKVRepeatedDiskExpandFallback
        expr: |
          sum by (cluster, shared_pool_id) (
            count_over_time(
              {cluster_env="prod", app="autoscaler-engine"}
                |= `OutputCapacityReservationV1.ExpandDisk.Fallback:`
                | regexp `'SharedPoolId' = (?P<shared_pool_id>[0-9]+);`
              [30m]
            )
          ) >= 2
        for: 1m
        labels:
          severity: major
          response_class: ticket
          tier: next-gen
          component: cluster-ng-service
          oncall_team: premium
        annotations:
          summary: "TiKV disk expansion repeatedly fell back to adding nodes"
          message: 'resource pool {{ $labels.shared_pool_id }} in {{ $labels.cluster }} entered disk-expand fallback at least twice in 30 minutes; repeated fallback can add unexpected TiKV nodes'
          runbook_url: https://github.com/tidbcloud/runbooks/wiki/TiKVRepeatedDiskExpandFallback