groups:
  - name: mgmt.tidb-mgmt-service
    rules:
      - alert: SyncToJSMFailed
        expr: |
          sum by (cluster_id, cluster_name, event_type) (
            increase(
              tidb_management_service_consumer_sync_to_jsm_failed_total{
                namespace="tidb-management-service",
                job="tidb-mgmt-service"
              }[5m]
            )
          ) > 0
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control
          service: tidb-mgmt-service
        annotations:
          message: Sync to JSM failed in last 5m cluster_id={{ $labels.cluster_id }} cluster_name={{ $labels.cluster_name }} event_type={{ $labels.event_type }} value={{ $value }}

      - alert: ClusterAutoResumed
        expr: |
          sum by (cluster_id, cluster_name, project_id, org_id) (
            increase(
              tidb_management_service_auto_resume_cluster_count_total{
                namespace="tidb-management-service",
                job="tidb-mgmt-service"
              }[5m]
            )
          ) > 0
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control
          service: tidb-mgmt-service
        annotations:
          message: Cluster auto-resumed in last 5m cluster_id={{ $labels.cluster_id }} cluster_name={{ $labels.cluster_name }} project_id={{ $labels.project_id }} org_id={{ $labels.org_id }} value={{ $value }}

      # Pool. Prepared instances < 20% of min_capacity
      - alert: ShadowPoolLowPreparedCapacity
        expr: |
          max by(pool_id) (tidb_management_service_shadow_pool_cluster_status{shadow_state="prepared", is_deleted="false"})
          < 
          on(pool_id) group_left() max by(pool_id) (tidb_management_service_shadow_pool_metadata_cluster{capacity_type="min_capacity"} * 0.2)
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control
          service: tidb-mgmt-service
        annotations:
          summary: "Low prepared capacity in pool {{ $labels.pool_id }}"
          description: "The number of 'prepared' instances is {{ $value }}, which is below 50% of the minimum capacity."

      # Pool. Total instances (including deleted) > 200000
      - alert: ShadowPoolMaxInstanceLimitReached
        expr: |
          sum(max by(pool_id, shadow_state) (tidb_management_service_shadow_pool_cluster_status{is_deleted="true"})) > 200000
        for: 2m
        labels:
          severity: critical
          response_class: page
          component: tidb-management-service
          oncall_team: global_control
          service: tidb-mgmt-service
        annotations:
          summary: "Pool {{ $labels.pool_id }} instance count critical"
          description: "Total instances (including deleted) reached {{ $value }}. This may indicate a resource leak or excessive churn."

      # Pool. Last fulfill quantity >= 10
      - alert: ShadowPoolLargeFulfillBatch
        expr: |
          sum by(pool_id) (increase(tidb_management_service_shadow_pool_fulfill_quantity_total[1m])) >= 10
        for: 0m
        labels:
          severity: warning
          component: tidb-management-service
          oncall_team: global_control
          service: tidb-mgmt-service
        annotations:
          summary: "Large fulfill operation detected in pool {{ $labels.pool_id }}"
          description: "Fulfill quantity is {{ $value }} in the last minute. This indicates a sudden burst of instance creation."

      # Pool. Rent request failures > 10 within 1 minutes
      - alert: ShadowPoolRentRequestFailuresHigh
        expr: |
          sum by(pool_id) (increase(tidb_management_service_shadow_pool_api_require_database_failure_total[10m])) > 10
        for: 1m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control
          service: tidb-mgmt-service
        annotations:
          summary: "High rent failure rate for pool {{ $labels.pool_id }}"
          description: "Rent requests failed {{ $value }} times over the last 10 minutes."

      # Pool. Average rent latency > 3s over 10 minutes
      - alert: ShadowPoolRentLatencyHigh
        expr: |
          (sum by(pool_id) (rate(tidb_management_service_shadow_pool_api_rent_cluster_duration_seconds_sum[10m])) 
          / 
          sum by(pool_id) (rate(tidb_management_service_shadow_pool_api_rent_cluster_duration_seconds_count[10m]))) > 3
        for: 1m
        labels:
          severity: major
          response_class: ticket
          component: tidb-management-service
          oncall_team: global_control
          service: tidb-mgmt-service
        annotations:
          summary: "High rent latency for pool {{ $labels.pool_id }}"
          description: "The average rent latency is {{ $value | printf \"%.2f\" }}s over the last 10 minutes (threshold: 3s)."