groups:
  - name: drive9
    rules:
      # Pre-aggregate the high-cardinality Drive9 counters once per evaluation.
      # The alerts below reuse these smaller result sets for the error ratio and
      # volume floor instead of scanning the same raw counters three times.
      - record: drive9_service_operations:rate5m:sum_by_namespace_component_result
        expr: |
          sum by (namespace, component, result) (
            rate(drive9_service_operations_total{component!~"sse|s3client|webdav|image_extract|audio_extract|video_extract|semantic_worker|mutation_replay|file_gc|expiry_sweep|event_bus|embedding|encrypt|leader|tenant_worker"}[5m])
          )

      - record: drive9_tenant_requests:rate5m:sum_by_namespace_tenant_surface_status
        expr: |
          sum by (namespace, tenant_id, tidbcloud_org_id, surface, status_class) (
            rate(drive9_tenant_requests_total{surface=~"fs|upload|object_store|sql|events|journal|git_workspace"}[5m])
          )

      # Per-(route, method) 5xx ratio — HIGH. All HTTP error-rate alerting is done
      # per (route, method) rather than fleet-wide: a global ratio hides a single
      # endpoint/method failing hard (e.g. all PUT /v1/fs/* writes 5xx) when that
      # route is a small share of total traffic, so low-traffic / niche APIs would
      # go unnoticed. A low per-series volume floor (10 requests over the window)
      # keeps niche routes covered while still filtering out single-request noise.
      # Major: localized degradation.
      - alert: Drive9RouteHTTPErrorRateHigh
        expr: |
          (
            (
              sum by (namespace, route, method) (rate(drive9_http_requests_total{route!~"/healthz|/metrics|other",status=~"5.."}[5m]))
              /
              clamp_min(sum by (namespace, route, method) (rate(drive9_http_requests_total{route!~"/healthz|/metrics|other"}[5m])), 0.000001)
            ) > 0.10
          )
          and
          (
            sum by (namespace, route, method) (increase(drive9_http_requests_total{route!~"/healthz|/metrics|other"}[5m])) >= 10
          )
        for: 2m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 per-route HTTP 5xx ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) {{ $labels.method }} {{ $labels.route }} 5xx ratio exceeded 10% over 5 minutes. Current value: {{ $value | humanizePercentage }}"

      # Per-(route, method) 5xx ratio — CRITICAL. Same per-route grouping as the
      # high alert above so a niche endpoint failing outright pages at critical
      # instead of being masked by a fleet-wide ratio. Shorter 5m window + lower
      # `for` so a hard per-route outage escalates quickly; the low per-series
      # volume floor (10 requests over the window) still filters single-request
      # noise. A broad outage trips this for every high-traffic route simultaneously.
      - alert: Drive9RouteHTTPErrorRateCritical
        expr: |
          (
            (
              sum by (namespace, route, method) (rate(drive9_http_requests_total{route!~"/healthz|/metrics|other",status=~"5.."}[5m]))
              /
              clamp_min(sum by (namespace, route, method) (rate(drive9_http_requests_total{route!~"/healthz|/metrics|other"}[5m])), 0.000001)
            ) > 0.30
          )
          and
          (
            sum by (namespace, route, method) (increase(drive9_http_requests_total{route!~"/healthz|/metrics|other"}[5m])) >= 10
          )
        for: 2m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 per-route HTTP 5xx ratio is critical"
          description: "Drive9 (ns={{ $labels.namespace }}) {{ $labels.method }} {{ $labels.route }} 5xx ratio exceeded 30% over 5 minutes. Current value: {{ $value | humanizePercentage }}"

      # High route-specific p99 latency on the main Drive9 HTTP surface.
      # Differentiates read vs write: write paths (PUT/POST/PATCH/DELETE) have
      # a higher threshold because large-file uploads naturally take longer.
      - alert: Drive9HighHTTPP99Latency
        expr: |
          (
            histogram_quantile(0.99,
              sum(rate(drive9_http_request_duration_seconds_bucket{route!~"/healthz|/metrics|other",method=~"GET|HEAD"}[5m])) by (namespace, le, route, method)
            ) > 2
          )
          or
          (
            histogram_quantile(0.99,
              sum(rate(drive9_http_request_duration_seconds_bucket{route!~"/healthz|/metrics|other",method=~"PUT|POST|PATCH|DELETE"}[5m])) by (namespace, le, route, method)
            ) > 10
          )
        for: 2m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 HTTP p99 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) {{ $labels.method }} {{ $labels.route }} p99 latency exceeded threshold over the past 5 minutes. Current value: {{ $value | humanizeDuration }}"

      # Critical route-specific p99 latency on the main Drive9 HTTP surface.
      - alert: Drive9CriticalHTTPP99Latency
        expr: |
          (
            histogram_quantile(0.99,
              sum(rate(drive9_http_request_duration_seconds_bucket{route!~"/healthz|/metrics|other",method=~"GET|HEAD"}[5m])) by (namespace, le, route, method)
            ) > 5
          )
          or
          (
            histogram_quantile(0.99,
              sum(rate(drive9_http_request_duration_seconds_bucket{route!~"/healthz|/metrics|other",method=~"PUT|POST|PATCH|DELETE"}[5m])) by (namespace, le, route, method)
            ) > 20
          )
        for: 2m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 HTTP p99 latency is critical"
          description: "Drive9 (ns={{ $labels.namespace }}) {{ $labels.method }} {{ $labels.route }} p99 latency exceeded threshold over the past 5 minutes. Current value: {{ $value | humanizeDuration }}"

      # Backend write latency after the HTTP request body has already been read.
      # This separates server/backend write pressure from client upload or network-path slowness.
      - alert: Drive9FSBackendWriteP99High
        expr: |
          histogram_quantile(0.99,
            sum(rate(drive9_service_operation_duration_seconds_bucket{component="backend",operation="write"}[5m])) by (namespace, le)
          ) > 2
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 filesystem backend write p99 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) backend write p99 latency after request-body read exceeded 2 seconds for 5 minutes. Current value: {{ $value | humanizeDuration }}"

      # Inline PUT request-body read latency. High values usually indicate slow client upload,
      # client-side contention, or network-path issues before the backend write begins.
      - alert: Drive9FSUploadBodyReadP99High
        expr: |
          histogram_quantile(0.99,
            sum(rate(drive9_http_request_body_read_duration_seconds_bucket{method="PUT",route="/v1/fs/*"}[5m])) by (namespace, le)
          ) > 5
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 filesystem PUT request-body read p99 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) inline PUT request-body read p99 latency exceeded 5 seconds for 5 minutes. Treat this as a client upload or network-path signal before attributing the incident to backend write latency."

      # Metadata DB pool waits indicate control-plane requests are blocking on
      # the per-pod meta sql.DB pool. Meta uses a small number of common pools,
      # so sustained waits usually mean the configured meta max-open limit is
      # too low or the metadata database is slow.
      - alert: Drive9MetaDBPoolWaitRateHigh
        expr: |
          sum by (namespace) (rate(drive9_db_pool_wait_count_total{role="meta"}[5m])) > 0.2
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 metadata DB pool wait rate is high"
          description: "Drive9 (ns={{ $labels.namespace }}) metadata DB pool has been waiting on sql.DB connections at more than 0.2 waits/sec for 5 minutes. Current value: {{ $value }} waits/sec"

      # Tenant user DB waits are tracked per cached tenant pool. Keep this
      # threshold less sensitive than metadata DB waits to avoid paging on small
      # transient bursts, but preserve tenant_id for attribution.
      - alert: Drive9UserDBPoolWaitRateHigh
        expr: |
          sum by (namespace, tenant_id, tidbcloud_org_id) (rate(drive9_db_pool_wait_count_total{role="user"}[5m])) > 1
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant DB pool wait rate is high"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} user DB pool has been waiting on sql.DB connections at more than 1 waits/sec for 5 minutes. Current value: {{ $value }} waits/sec"

      # Shared DB handles expose their globally unique DB-pool UUID directly;
      # they do not use the tenant_id label because one handle serves many tenants.
      - alert: Drive9SharedDBPoolWaitRateHigh
        expr: |
          sum by (namespace, db_pool_uuid, tidbcloud_org_id) (
            rate(drive9_db_pool_wait_count_total{role="shared"}[5m])
          ) > 1
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 shared DB pool wait rate is high"
          description: "Drive9 (ns={{ $labels.namespace }}) shared DB pool {{ $labels.db_pool_uuid }} / TiDB org {{ $labels.tidbcloud_org_id }} has been waiting on sql.DB connections at more than 1 wait/sec for 5 minutes. All tenants placed on this physical DB pool may be affected. Current value: {{ $value }} waits/sec"

      # Each pod owns an independent LRU cache of tenant user DB backends. Alert
      # before capacity eviction starts causing cold opens and connection churn.
      - alert: Drive9UserDBLRUCapacityCritical
        expr: |
          (
            max by (namespace, pod) (drive9_service_gauge{component="tenant_pool",name="cached_backends"})
            /
            clamp_min(max by (namespace, pod) (drive9_service_gauge{component="tenant_pool",name="max_backends"}), 1)
          ) > 0.80
          and on (namespace, pod)
          (
            max by (namespace, pod) (drive9_service_gauge{component="tenant_pool",name="max_backends"}) > 0
          )
        for: 3m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 user DB LRU cache is near capacity"
          description: "Drive9 (ns={{ $labels.namespace }}) pod {{ $labels.pod }} user DB LRU cache has exceeded 80% of its configured capacity for 3 minutes. Current usage: {{ $value | humanizePercentage }}"

      # A failed physical shared DB pool means its reserved tenants cannot
      # become active until provisioning is retried or replacement capacity is
      # created. Each series identifies one physical pool by its global UUID.
      - alert: Drive9SharedDBPoolProvisionFailed
        expr: |
          max by (namespace, db_pool_id, db_pool_uuid, tidbcloud_org_id) (
            drive9_shared_db_pool_total{status="failed"}
          ) > 0
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 shared DB pool provisioning has failed"
          description: "Drive9 (ns={{ $labels.namespace }}) shared DB pool {{ $labels.db_pool_uuid }} (local ID {{ $labels.db_pool_id }}) / TiDB org {{ $labels.tidbcloud_org_id }} has remained failed for 5 minutes. Reserved tenants may remain provisioning until replacement capacity is created."

      # Crossing the soft capacity is an intentional emergency fallback when
      # new physical DB pools cannot be provisioned. Alert only near the hard
      # cap, where the remaining emergency headroom is almost exhausted.
      - alert: Drive9SharedDBPoolHardCapacityHigh
        expr: |
          (
            max by (namespace, db_pool_id, db_pool_uuid, tidbcloud_org_id) (
              drive9_shared_db_pool_capacity{type="used"}
            )
            /
            clamp_min(
              max by (namespace, db_pool_id, db_pool_uuid, tidbcloud_org_id) (
                drive9_shared_db_pool_capacity{type="hard_max"}
              ),
              1
            )
          ) > 0.90
          and
          max by (namespace, db_pool_id, db_pool_uuid, tidbcloud_org_id) (
            drive9_shared_db_pool_capacity{type="hard_max"}
          ) > 0
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 shared DB pool is near hard capacity"
          description: "Drive9 (ns={{ $labels.namespace }}) shared DB pool {{ $labels.db_pool_uuid }} (local ID {{ $labels.db_pool_id }}) / TiDB org {{ $labels.tidbcloud_org_id }} has exceeded 90% of its emergency hard tenant capacity for 5 minutes. Provision or recover another physical shared DB pool before this pool becomes unable to accept tenants. Current usage: {{ $value | humanizePercentage }}"

      # A cached shared DB handle with zero tenant-backend refs should be
      # closed by the idle reaper after the shared-handle TTL (30m, reaped
      # every 2m). One that survives well past the TTL means the reaper is
      # stuck or the handle is pinned. Handle count alone is NOT a leak
      # signal: under uniform traffic every pod legitimately warms handles to
      # most physical pools, so only a zero-ref handle outliving its TTL is
      # actionable. Ref-count leaks (refs stuck >0) are intentionally not
      # covered here; they need a separate observable condition.
      - alert: Drive9SharedDBPoolCacheHandleStuck
        expr: |
          drive9_shared_db_pool_cache_handles == 1
          and on (namespace, pod, db_pool_id, db_pool_uuid, tidbcloud_org_id)
          drive9_shared_db_pool_cache_tenants == 0
        for: 45m
        labels:
          severity: warning
          response_class: logging
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 shared DB handle idle past reaper TTL"
          description: "Drive9 (ns={{ $labels.namespace }}) pod {{ $labels.pod }} has kept a cached sql.DB handle on shared DB pool {{ $labels.db_pool_uuid }} (local ID {{ $labels.db_pool_id }}) / TiDB org {{ $labels.tidbcloud_org_id }} with zero tenant-backend refs for 45 minutes, past the 30-minute shared-handle idle TTL. The idle reaper should have closed it; this indicates a stuck reaper or a pinned handle."

      # Tenant schema-init DB waits mean schema bootstrap is being throttled by
      # a tenant schema sql.DB pool. This role is intentionally separate
      # from request-path user pools because schema init runs DDL and has different
      # latency characteristics.
      - alert: Drive9UserSchemaDBPoolWaitRateHigh
        expr: |
          sum by (namespace, tenant_id, tidbcloud_org_id) (rate(drive9_db_pool_wait_count_total{role="user_schema"}[5m])) > 0.2
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant schema DB pool wait rate is high"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} schema-init DB pool has been waiting on sql.DB connections at more than 0.2 waits/sec for 5 minutes. Current value: {{ $value }} waits/sec"

      # Metadata DB operation latency captures control-plane store pressure even
      # before full request failures appear.
      - alert: Drive9MetaDBOperationP99High
        expr: |
          histogram_quantile(0.99,
            sum(rate(drive9_db_operation_duration_seconds_bucket{role="meta"}[5m])) by (namespace, le)
          ) > 3
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 metadata DB operation p99 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) metadata DB p99 operation latency exceeded 3 seconds over the past 5 minutes. Current value: {{ $value | humanizeDuration }}"

      # User DB operation latency is intentionally aggregated by role/namespace:
      # hot-path DB operation metrics do not carry tenant_id to avoid Prometheus
      # cardinality blow-up. Use request logs and tenant-scoped worker/service
      # metrics to identify affected tenants.
      - alert: Drive9UserDBOperationP99High
        expr: |
          histogram_quantile(0.99,
            sum(rate(drive9_db_operation_duration_seconds_bucket{role="user"}[5m])) by (namespace, le)
          ) > 2.5
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 user DB operation p99 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) aggregate user DB p99 operation latency exceeded 2.5 seconds over the past 5 minutes. This alert is role-level; use request logs and tenant-scoped worker/service metrics to identify affected tenants. Current value: {{ $value | humanizeDuration }}"

      # Shared DB operations serve multiple tenants through each physical DB
      # pool. Operation histograms are intentionally role-level; use the shared
      # DB pool wait/capacity alerts and logs to identify the physical pool.
      - alert: Drive9SharedDBOperationP99High
        expr: |
          histogram_quantile(0.99,
            sum(rate(drive9_db_operation_duration_seconds_bucket{role="shared"}[5m])) by (namespace, le)
          ) > 2.5
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 shared DB operation p99 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) aggregate shared DB p99 operation latency exceeded 2.5 seconds over the past 5 minutes. Multiple tenants on one or more physical shared DB pools may be affected; use shared DB pool wait metrics and logs for attribution. Current value: {{ $value | humanizeDuration }}"

      # Metadata DB latency is a control-plane early warning. The meta DB p99
      # alert above catches severe degradation; this meta-specific
      # p95 rule fires earlier so auth, tenant routing, provisioning, leader election,
      # and admin tenant-pool paths can be investigated before requests start failing.
      - alert: Drive9MetaDBOperationP95High
        expr: |
          histogram_quantile(0.95,
            sum(rate(drive9_db_operation_duration_seconds_bucket{role="meta"}[5m])) by (namespace, le)
          ) > 1
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 metadata DB p95 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) metadata DB p95 operation latency exceeded 1 second for 5 minutes. Current value: {{ $value | humanizeDuration }}"

      # Metadata (control-plane) DB unreachable: every pod failed to ping the meta store
      # on its last health probe. This is independent of live traffic, so it fires even
      # for an idle-but-down metadb. With the control-plane DB gone, provisioning, auth,
      # tenant routing, and leader election all stop -> full service outage.
      - alert: Drive9MetaDBUnavailable
        expr: |
          max by (namespace) (drive9_db_up{role="meta"}) == 0
        for: 2m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 metadata database is unavailable"
          description: "Drive9 (ns={{ $labels.namespace }}) control-plane (meta) database has failed health probes on every pod for 2 minutes. Provisioning, auth, and tenant routing are unavailable."

      # Connection-pool saturation (golden signal): in-use connections near the configured
      # max. Guarded to pods that set an explicit max (max_open > 0); unlimited pools report
      # max_open = 0 and are excluded. Complements the role-specific pool-wait alerts,
      # which only move once requests are already blocking on the pool.
      - alert: Drive9DBConnectionPoolSaturation
        expr: |
          (
            (
              drive9_db_pool_connections{state="in_use"}
              / ignoring(state)
              clamp_min(drive9_db_pool_connections{state="max_open"}, 1)
            ) > 0.90
          )
          and ignoring(state)
          (
            drive9_db_pool_connections{state="max_open"} > 0
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 DB connection pool is near saturation"
          description: "Drive9 (ns={{ $labels.namespace }}) DB role {{ $labels.role }} in-use connections have stayed above 90% of the configured max for 5 minutes. Current value: {{ $value | humanizePercentage }}"

      # Elevated metadata DB error/bad-conn ratio under live traffic: catches a
      # degraded (not fully down) metadata database that still answers health
      # pings but fails real operations.
      - alert: Drive9MetaDBOperationErrorRateHigh
        expr: |
          (
            (
              sum by (namespace) (rate(drive9_db_operations_total{role="meta",result=~"error|bad_conn"}[5m]))
              /
              clamp_min(sum by (namespace) (rate(drive9_db_operations_total{role="meta"}[5m])), 0.000001)
            ) > 0.10
          )
          and
          (
            sum by (namespace) (increase(drive9_db_operations_total{role="meta"}[5m])) > 50
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 metadata DB operation error ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) metadata DB operation errors exceeded 10% over 5 minutes. Current value: {{ $value | humanizePercentage }}"

      # Central quota reservation lock conflicts should usually be absorbed by
      # bounded retry. This normal retry outcome is aggregate-only to avoid one
      # success-path series per tenant.
      - alert: Drive9QuotaReservationLockConflictRetryHigh
        expr: |
          sum by (namespace) (
            increase(drive9_service_operations_total{component="central_quota",operation="reserve_upload",result="lock_conflict_retry"}[5m])
          ) > 20
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 central quota reservation lock-conflict retry volume is high"
          description: "Drive9 (ns={{ $labels.namespace }}) had more than 20 central quota upload-reservation lock-conflict retries over a 5 minute window for 5 minutes. Uploads may still be succeeding after retry, but quota admission is under hot-row contention and may fail open if retry is exhausted. Current retry count: {{ $value }}"

      # busy_fail_open means the server exhausted bounded lock-conflict retry
      # but preserved upload availability by fail-opening around quota admission.
      # Sustained occurrences indicate quota hot-row pressure and a temporary
      # admission-accounting gap for in-flight uploads.
      - alert: Drive9QuotaReservationBusyFailOpen
        expr: |
          sum by (namespace, tenant_id, tidbcloud_org_id) (
            increase(drive9_tenant_operation_failures_total{component="central_quota",operation="reserve_upload",result="busy_fail_open"}[5m])
          ) >= 5
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 central quota reservation is busy and fail-open"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} repeatedly exhausted bounded central quota upload-reservation lock-conflict retry. The upload initiate path preserved availability by fail-opening around quota admission, so in-flight uploads may be under-represented in reserved_bytes. Current 5m busy_fail_open count: {{ $value }}"

      # User DB operation errors are intentionally aggregated by role/namespace:
      # hot-path DB operation metrics do not carry tenant_id to avoid Prometheus
      # cardinality blow-up. Use request logs and tenant-scoped worker/service
      # metrics for per-tenant attribution.
      - alert: Drive9UserDBOperationErrorRateHigh
        expr: |
          (
            (
              sum by (namespace) (rate(drive9_db_operations_total{role="user",result=~"error|bad_conn"}[5m]))
              /
              clamp_min(sum by (namespace) (rate(drive9_db_operations_total{role="user"}[5m])), 0.000001)
            ) > 0.10
          )
          and
          (
            sum by (namespace) (increase(drive9_db_operations_total{role="user"}[5m])) > 50
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 user DB operation error ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) aggregate user DB operation errors exceeded 10% over 5 minutes. This alert is role-level; use request logs and tenant-scoped worker/service metrics to identify affected tenants. Current value: {{ $value | humanizePercentage }}"

      # Shared DB operation failures have a larger blast radius than a
      # per-tenant user DB failure. Keep the same traffic floor and ratio as the
      # user role while reporting shared independently.
      - alert: Drive9SharedDBOperationErrorRateHigh
        expr: |
          (
            (
              sum by (namespace) (rate(drive9_db_operations_total{role="shared",result=~"error|bad_conn"}[5m]))
              /
              clamp_min(sum by (namespace) (rate(drive9_db_operations_total{role="shared"}[5m])), 0.000001)
            ) > 0.10
          )
          and
          (
            sum by (namespace) (increase(drive9_db_operations_total{role="shared"}[5m])) > 50
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 shared DB operation error ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) aggregate shared DB operation errors exceeded 10% over 5 minutes. Multiple tenants on one or more physical shared DB pools may be affected; use shared DB pool wait metrics and logs for attribution. Current value: {{ $value | humanizePercentage }}"

      # Semantic worker unavailable means async semantic task processing cannot run.
      - alert: Drive9SemanticWorkerUnavailable
        expr: |
          drive9_module_up{module="semantic_worker"} == 0
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 semantic worker is unavailable"
          description: "Drive9 (ns={{ $labels.namespace }}) semantic worker module has been unavailable for 5 minutes while the service is still exporting metrics."

      # Queue lag is emitted only for tenant-scoped abnormal state. Healthy and
      # dormant tenants delete the gauge instead of retaining a zero or stale value.
      - alert: Drive9SemanticWorkerTenantQueueLagHigh
        expr: |
          drive9_service_gauge{component="semantic_worker",name="queue_lag_seconds",tenant_id!="",tenant_id!="unknown"} > 300
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 observed semantic worker per-tenant queue lag is high"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} observed semantic queue lag exceeded 5 minutes for 5 minutes. This signal is emitted only while the tenant backend is local/warm cached. Current value: {{ $value | humanizeDuration }}"

      # Dead-letter accumulation usually means a provider or task handler is consistently failing.
      - alert: Drive9SemanticWorkerTenantDeadLettered
        expr: |
          drive9_service_gauge{component="semantic_worker",name="dead_lettered",tenant_id!="",tenant_id!="unknown"} > 0
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 observed semantic worker tenant dead-lettered tasks"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} has observed dead-lettered semantic tasks for 5 minutes. This signal is emitted only while the tenant backend is local/warm cached. Current value: {{ $value }} tasks"

      # Image, audio, and video extraction share the durable semantic-worker
      # contract. Group by task operation so one healthy extractor cannot dilute
      # a failing one.
      - alert: Drive9SemanticExtractionFailureRateHigh
        expr: |
          (
            (
              sum by (namespace, operation) (rate(drive9_service_operations_total{component="semantic_worker",operation=~"img_extract_text|audio_extract_text|video_extract_visual",result=~"error|.+_error|runtime_not_configured|handler_missing|unsupported|lease_lost|dead_lettered"}[5m]))
              /
              clamp_min(sum by (namespace, operation) (rate(drive9_service_operations_total{component="semantic_worker",operation=~"img_extract_text|audio_extract_text|video_extract_visual"}[5m])), 0.000001)
            ) > 0.20
          )
          and
          (
            sum by (namespace, operation) (increase(drive9_service_operations_total{component="semantic_worker",operation=~"img_extract_text|audio_extract_text|video_extract_visual"}[5m])) > 10
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 semantic extraction failure ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) semantic extraction task {{ $labels.operation }} failures exceeded 20% over 5 minutes. Current value: {{ $value | humanizePercentage }}"

      # Queue saturation usually shows up before image extraction fully stops; alert before tasks start dropping on queue_full.
      - alert: Drive9ImageExtractQueueSaturationHigh
        expr: |
          (
            drive9_service_gauge{component="image_extract",name="queue_depth"}
            / ignoring(name)
            clamp_min(drive9_service_gauge{component="image_extract",name="queue_capacity"}, 1)
          ) > 0.80
          and ignoring(name)
          drive9_service_gauge{component="image_extract",name="workers"} > 0
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 image extract queue is near saturation"
          description: "Drive9 (ns={{ $labels.namespace }}) image extraction queue depth has stayed above 80% of capacity for 5 minutes. Current value: {{ $value | humanizePercentage }}"

      # FUSE remote-path error ratio is only relevant when mounted clients are active.
      - alert: Drive9FuseRemoteErrorRateHigh
        expr: |
          (
            (
              sum by (namespace, operation) (rate(drive9_fuse_remote_operations_total{result!="ok"}[5m]))
              /
              clamp_min(sum by (namespace, operation) (rate(drive9_fuse_remote_operations_total[5m])), 0.000001)
            ) > 0.20
          )
          and
          (
            sum by (namespace, operation) (increase(drive9_fuse_remote_operations_total[5m])) > 20
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 FUSE remote-path error ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) FUSE remote operation {{ $labels.operation }} exceeded 20% errors over 5 minutes. Current value: {{ $value | humanizePercentage }}"

      # FUSE remote latency spikes often arrive before outright failures and directly impact mounted client UX.
      - alert: Drive9FuseRemoteOperationP95High
        expr: |
          histogram_quantile(0.95,
            sum(rate(drive9_fuse_remote_operation_duration_seconds_bucket[5m])) by (namespace, le, operation)
          ) > 2
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 FUSE remote operation p95 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) FUSE remote operation {{ $labels.operation }} p95 latency exceeded 2 seconds over 5 minutes. Current value: {{ $value | humanizeDuration }}"

      # Raw object-store failures are often invisible at HTTP level until upload / download paths start backing up.
      - alert: Drive9S3ClientErrorRateHigh
        expr: |
          (
            (
              sum by (namespace, operation) (rate(drive9_service_operations_total{component="s3client",result="error"}[5m]))
              /
              clamp_min(sum by (namespace, operation) (rate(drive9_service_operations_total{component="s3client"}[5m])), 0.000001)
            ) > 0.15
          )
          and
          (
            sum by (namespace, operation) (increase(drive9_service_operations_total{component="s3client"}[5m])) > 20
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 s3client error ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) s3client operation {{ $labels.operation }} exceeded 15% errors over 5 minutes. Current value: {{ $value | humanizePercentage }}"

      # S3 latency spikes often precede end-user upload / download symptoms; keep the threshold loose enough for multipart completion.
      - alert: Drive9S3ClientOperationP95High
        expr: |
          histogram_quantile(0.95,
            sum(rate(drive9_service_operation_duration_seconds_bucket{component="s3client"}[5m])) by (namespace, le, operation)
          ) > 2
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 s3client operation p95 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) s3client operation {{ $labels.operation }} p95 latency exceeded 2 seconds over 5 minutes. Current value: {{ $value | humanizeDuration }}"

      # WebDAV should not emit many server-side failures; conflicts and not-found are client-visible semantics and are excluded here.
      - alert: Drive9WebDAVErrorRateHigh
        expr: |
          (
            (
              sum by (namespace, operation) (rate(drive9_service_operations_total{component="webdav",result="error"}[5m]))
              /
              clamp_min(sum by (namespace, operation) (rate(drive9_service_operations_total{component="webdav"}[5m])), 0.000001)
            ) > 0.05
          )
          and
          (
            sum by (namespace, operation) (increase(drive9_service_operations_total{component="webdav"}[5m])) > 20
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 WebDAV error ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) WebDAV operation {{ $labels.operation }} exceeded 5% server-side errors over 5 minutes. Current value: {{ $value | humanizePercentage }}"

      # Per-tenant request ratios are valid only for data-plane surfaces, whose
      # success and failure series both carry tenant labels. Control-plane 2xx
      # requests are aggregate-only to bound cardinality, while their 4xx/5xx
      # series retain tenant labels for direct diagnosis.
      - alert: Drive9TenantServerErrorRateHigh
        expr: |
          (
            (
              sum by (namespace, tenant_id, tidbcloud_org_id, surface) (drive9_tenant_requests:rate5m:sum_by_namespace_tenant_surface_status{status_class="5xx"})
              /
              clamp_min(sum by (namespace, tenant_id, tidbcloud_org_id, surface) (drive9_tenant_requests:rate5m:sum_by_namespace_tenant_surface_status), 0.000001)
            ) > 0.1
          )
          and
          (
            sum by (namespace, tenant_id, tidbcloud_org_id, surface) (drive9_tenant_requests:rate5m:sum_by_namespace_tenant_surface_status) * 300 > 20
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant server error ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} surface {{ $labels.surface }} exceeded 10% 5xx responses over 5 minutes. Current value: {{ $value | humanizePercentage }}"

      # Invalid-path rejections are client input errors (HTTP 400), so they are
      # deliberately absent from every 5xx alert above. They still need a
      # dedicated signal, otherwise a server-side path-handling regression that
      # rejects valid paths would be invisible (400s do not page). The event is
      # emitted only when path validation rejects a request (e.g. a tenant
      # concatenating Windows local paths with drive letters/backslashes into
      # Drive9 paths), never for routine 404 probes, so any sustained volume is
      # actionable. Per-tenant warning: one client is misbehaving -> ask the
      # tenant to translate local paths before calling the API.
      - alert: Drive9FSInvalidPathRejectedTenant
        expr: |
          sum by (namespace, tenant_id, tidbcloud_org_id) (
            increase(drive9_business_events_total{event="fs_invalid_path",tenant_id!="",tenant_id!="unknown"}[15m])
          ) > 10
        for: 5m
        labels:
          severity: warning
          response_class: logging
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant is sending invalid fs paths"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} had more than 10 fs requests rejected for invalid paths (HTTP 400) over 15 minutes. This is almost always a client-side path translation bug (e.g. Windows drive letters/backslashes sent as Drive9 path segments); check mkdir_invalid_path / request logs for the exact paths and ask the tenant to convert local paths to Drive9 absolute POSIX paths. Current value: {{ $value }} rejections"

      # Aggregate fallback at higher severity: a cross-tenant spike of
      # invalid-path rejections is NOT explained by one misbehaving client and
      # suggests a server-side path-handling regression (our bug), so it
      # escalates to major.
      - alert: Drive9FSInvalidPathRejectedAggregate
        expr: |
          sum by (namespace) (
            increase(drive9_business_events_total{event="fs_invalid_path"}[15m])
          ) > 50
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 invalid fs path rejections are spiking across tenants"
          description: "Drive9 (ns={{ $labels.namespace }}) rejected more than 50 fs requests for invalid paths (HTTP 400) across tenants over 15 minutes. A single misbehaving client explains small per-tenant volumes, but a fleet-wide spike suggests a server-side path validation or routing regression; diff recent server releases and check mkdir_invalid_path / request logs. Current value: {{ $value }} rejections"

      # Storage quota pressure should be visible before writes start failing.
      - alert: Drive9TenantStorageQuotaNearLimit
        expr: |
          (
            (
              sum by (namespace, tenant_id, tidbcloud_org_id) (drive9_tenant_storage_bytes{state=~"confirmed|reserved"})
              /
              sum by (namespace, tenant_id, tidbcloud_org_id) (drive9_tenant_storage_bytes{state="limit"})
            ) > 0.90
          )
          and on (namespace, tenant_id, tidbcloud_org_id)
          (
            sum by (namespace, tenant_id, tidbcloud_org_id) (drive9_tenant_storage_bytes{state="limit"}) > 0
          )
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant storage quota is near limit"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} storage usage has exceeded 90% of its configured limit for 15 minutes. Current value: {{ $value | humanizePercentage }}"

      # Provisioning failures block new tenants from becoming usable.
      # This per-tenant variant fires when a specific tenant's provisioning fails.
      - alert: Drive9TenantProvisionFailures
        expr: |
          sum by (namespace, provider, result, tenant_id, tidbcloud_org_id) (
            increase(drive9_business_events_total{event="tenant_provision",result=~"error|cluster_error",tenant_id!="",tenant_id!="unknown"}[15m])
          ) > 2
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant provisioning is failing"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} provisioning has recorded repeated {{ $labels.result }} outcomes for provider {{ $labels.provider }} over the past 15 minutes. Current value: {{ $value }} failures"

      # Aggregate fallback: catches pre-tenant provisioning failures (e.g. credential
      # validation, unsupported provider) where no tenant_id has been assigned yet.
      - alert: Drive9TenantProvisionFailuresAggregate
        expr: |
          sum by (namespace, provider, result) (
            increase(drive9_business_events_total{event="tenant_provision",result=~"error|cluster_error"}[15m])
          ) > 3
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant provisioning is failing (aggregate)"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant provisioning has recorded repeated {{ $labels.result }} outcomes for provider {{ $labels.provider }} over the past 15 minutes. Current value: {{ $value }} failures"

      # Schema init failures block tenants from ever leaving provisioning. Ignore
      # tenants that recovered within the same window: TiDB native clusters often
      # return transient "No available TiDB instances", auth, or DDL schema-change
      # errors before a later retry succeeds.
      - alert: Drive9TenantSchemaInitFailures
        expr: |
          (
            sum by (namespace, provider, tenant_id, tidbcloud_org_id) (
              increase(drive9_business_events_total{event="tenant_schema_init",result="error",tenant_id!="",tenant_id!="unknown"}[15m])
            ) >= 3
          )
          unless on (namespace, provider, tenant_id, tidbcloud_org_id)
          (
            sum by (namespace, provider, tenant_id, tidbcloud_org_id) (
              increase(drive9_business_events_total{event="tenant_schema_init",result="ok",tenant_id!="",tenant_id!="unknown"}[15m])
            ) > 0
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant schema initialization is failing"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} schema initialization has failed for provider {{ $labels.provider }} within the past 15 minutes. Current value: {{ $value }} failures"

      # Tenant-pool lifecycle operations call TiDB Cloud and can fail independently
      # from public provision traffic (for example 429/rate limits or deprovision
      # failures). Alert by operation so create/update/delete/replenish are visible.
      - alert: Drive9AdminTenantPoolOperationFailures
        expr: |
          sum by (namespace, operation, result) (
            increase(drive9_service_operations_total{component="admin_tenant_pool",operation=~"create|update|delete|replenish",result=~"error|cluster_error|deadline_exceeded|bad_conn"}[15m])
          ) > 10
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 admin tenant pool operation is failing"
          description: "Drive9 (ns={{ $labels.namespace }}) admin tenant pool operation {{ $labels.operation }} recorded repeated {{ $labels.result }} outcomes over the past 15 minutes. This usually means pool create/update/delete/replenish is blocked by TiDB Cloud API failures, rate limits, or metadata-store errors. Current value: {{ $value }} failures"

      # Metadata resume is the recovery path for tenant-pool clusters whose TiDB Cloud
      # batch-create response omitted connection fields. A batch wait failure or
      # deadline means pending pool tenants can remain unusable until another
      # credential-backed admin create/pool action retriggers resume.
      - alert: Drive9AdminTenantPoolMetadataResumeWaitFailures
        expr: |
          sum by (namespace, pool_id, organization_id, result) (
            increase(drive9_tenant_pool_metadata_resume_wait_total{scope="batch",result=~"deadline_exceeded|bad_conn|error"}[15m])
          ) > 5
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 admin tenant pool metadata resume wait is failing"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant pool {{ $labels.pool_id }} / TiDB org {{ $labels.organization_id }} metadata resume wait recorded {{ $labels.result }} over the past 15 minutes. Pending pool tenants may be stuck waiting for TiDB Cloud connection metadata. Current value: {{ $value }} failures"

      # Track the TiDB Cloud Cluster, IAM, and Billing APIs independently so a
      # low-volume failing operation is not diluted by healthy calls elsewhere.
      # Client errors and caller cancellation stay visible without paging.
      - alert: Drive9TiDBCloudOpenAPIErrorRateHigh
        expr: |
          (
            (
              sum by (namespace, api, operation) (
                rate(drive9_tidbcloud_openapi_requests_total{result=~"rate_limited|upstream_error|protocol_error|digest_error|timeout|transport_error"}[5m])
              )
              /
              clamp_min(
                sum by (namespace, api, operation) (
                  rate(drive9_tidbcloud_openapi_requests_total[5m])
                ),
                0.000001
              )
            ) > 0.20
          )
          and
          (
            sum by (namespace, api, operation) (
              increase(drive9_tidbcloud_openapi_requests_total{result=~"rate_limited|upstream_error|protocol_error|digest_error|timeout|transport_error"}[5m])
            ) >= 3
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 TiDB Cloud OpenAPI error ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) TiDB Cloud {{ $labels.api }}/{{ $labels.operation }} alerting error ratio exceeded 20% over 5 minutes with at least 3 errors. Current value: {{ $value | humanizePercentage }}"

      # A full dispatcher shard blocks quota mutation producers under the
      # mutation lock, directly stalling user writes.
      - alert: Drive9MutationDispatcherQueueSaturationHigh
        expr: |
          (
            drive9_mutation_dispatcher_queue_depth
            /
            clamp_min(drive9_mutation_dispatcher_queue_capacity, 1)
          ) > 0.80
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 mutation dispatcher queue is near saturation"
          description: "Drive9 (ns={{ $labels.namespace }}) pod {{ $labels.pod }} mutation dispatcher shard {{ $labels.shard }} has stayed above 80% capacity for 5 minutes. User writes are close to blocking. Current value: {{ $value | humanizePercentage }}"

      - alert: Drive9MutationDispatcherEnqueueBlockedCritical
        expr: |
          (
            histogram_quantile(0.95,
              sum by (namespace, pod, shard, le) (
                rate(drive9_mutation_dispatcher_enqueue_blocked_seconds_bucket[5m])
              )
            ) > 1
          )
          and
          (
            sum by (namespace, pod, shard) (
              increase(drive9_mutation_dispatcher_enqueue_blocked_seconds_count[5m])
            ) > 20
          )
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 mutation dispatcher enqueue is blocking writes"
          description: "Drive9 (ns={{ $labels.namespace }}) pod {{ $labels.pod }} mutation dispatcher shard {{ $labels.shard }} enqueue p95 exceeded 1 second with sustained traffic. User write paths are blocked behind quota mutation delivery. Current value: {{ $value | humanizeDuration }}"

      - alert: Drive9MutationDispatcherBatchFallbackHigh
        expr: |
          sum by (namespace) (increase(drive9_mutation_dispatcher_batch_fallback_total[15m])) > 3
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 mutation dispatcher batch fallback is high"
          description: "Drive9 (ns={{ $labels.namespace }}) repeatedly fell back from batch quota mutations to per-item transactions. Check for cross-pod deadlocks, poison mutations, and meta DB transaction failures. Current value: {{ $value }} fallbacks"

      # Authentication cache misses fall through to the meta DB. A sustained
      # miss ratio regression can restore the query volume the cache removed.
      - alert: Drive9APIKeyResolveCacheHitRateLow
        expr: |
          (
            sum by (namespace) (rate(drive9_api_key_resolve_cache_requests_total{result="miss"}[5m]))
            /
            clamp_min(sum by (namespace) (rate(drive9_api_key_resolve_cache_requests_total[5m])), 0.000001)
          ) > 0.20
          and
          sum by (namespace) (increase(drive9_api_key_resolve_cache_requests_total[5m])) > 100
        for: 10m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 API key resolve cache hit rate is low"
          description: "Drive9 (ns={{ $labels.namespace }}) API key resolve cache miss ratio exceeded 20% with sufficient traffic for 10 minutes. The authentication hot path may be overloading the meta DB. Current miss ratio: {{ $value | humanizePercentage }}"

      - alert: Drive9NotifyCoalescerFallbackHigh
        expr: |
          sum by (namespace) (increase(drive9_notify_coalescer_flush_total{result="fallback"}[15m])) > 3
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 notify coalescer repeatedly falls back to per-row inserts"
          description: "Drive9 (ns={{ $labels.namespace }}) notify coalescer repeatedly failed batch insertion and used per-row meta DB writes. Cross-pod notifications are degraded and meta DB load is elevated. Current value: {{ $value }} fallbacks"

      - alert: Drive9NotifyCoalescerPerRowFailures
        expr: |
          sum by (namespace) (increase(drive9_notify_coalescer_per_row_fallback_total{result="error"}[15m])) > 0
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 notify coalescer fallback inserts are failing"
          description: "Drive9 (ns={{ $labels.namespace }}) failed per-row tenant notification inserts after batch retry was exhausted. Cross-pod SSE and worker delivery can be lost. Current value: {{ $value }} failures"

      - alert: Drive9NotifyCoalescerPendingHigh
        expr: |
          max by (namespace, pod) (drive9_notify_coalescer_pending) > 1000
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 notify coalescer pending queue is high"
          description: "Drive9 (ns={{ $labels.namespace }}) pod {{ $labels.pod }} has more than 1000 tenant notifications pending for 5 minutes. Flush progress is stalled. Current value: {{ $value }} tenants"

      - alert: Drive9TenantOutboxBacklogAgeHigh
        expr: |
          max by (namespace, pod) (drive9_tenant_outbox_backlog_oldest_age_seconds) > 30
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant outbox delivery backlog is stale"
          description: "Drive9 (ns={{ $labels.namespace }}) pod {{ $labels.pod }} tenant outbox backlog has remained older than 30 seconds for 5 minutes. Cross-pod SSE and background work delivery are severely delayed. Current age: {{ $value | humanizeDuration }}"

      - alert: Drive9TenantOutboxPollFailures
        expr: |
          sum by (namespace) (
            increase(drive9_tenant_outbox_poll_duration_seconds_count{result!="ok"}[15m])
          ) > 3
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant outbox polling is failing"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant outbox poller recorded more than 3 meta DB read failures over 15 minutes. Cross-pod event and worker dispatch is degraded. Current value: {{ $value }} failures"

      - alert: Drive9TenantOutboxCursorFlushFailures
        expr: |
          sum by (namespace) (increase(drive9_tenant_outbox_cursor_flush_total{result!="ok"}[15m])) > 3
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant outbox cursor persistence is failing"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant outbox cursor repeatedly failed to persist. Pod restarts can replay more work and delay convergence. Current value: {{ $value }} failures"

      - alert: Drive9SharedDBPoolStatusStuck
        expr: |
          max by (namespace, db_pool_uuid, tidbcloud_org_id, status) (
            drive9_shared_db_pool_status_age_seconds{status=~"pending|provisioning"}
          ) > 600
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 shared DB pool status is not converging"
          description: "Drive9 (ns={{ $labels.namespace }}) shared pool {{ $labels.db_pool_uuid }} / TiDB org {{ $labels.tidbcloud_org_id }} has remained {{ $labels.status }} for more than 10 minutes and did not converge for another 5 minutes. Current age: {{ $value | humanizeDuration }}"

      - alert: Drive9SharedDBPoolStuckFailedHigh
        expr: |
          sum by (namespace, previous_status) (increase(drive9_shared_db_pool_stuck_marked_failed_total[30m])) > 3
        for: 0m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 frequently marks stuck shared pools failed"
          description: "Drive9 (ns={{ $labels.namespace }}) marked more than 3 shared pools failed from {{ $labels.previous_status }} over 30 minutes. Persistent pool creation or provisioning convergence is broken. Current value: {{ $value }} pools"

      - alert: Drive9SharedDBPoolWaveFailures
        expr: |
          sum by (namespace, result) (increase(drive9_shared_db_pool_wave_total{result!="ok"}[15m])) > 0
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 shared pool reservation wave is failing"
          description: "Drive9 (ns={{ $labels.namespace }}) shared pool reservation wave recorded {{ $labels.result }} in the past 15 minutes. Persistent pool replenishment may not converge. Current value: {{ $value }} failures"

      - alert: Drive9SharedDBPoolCleanupFailures
        expr: |
          sum by (namespace, stage, result) (
            increase(drive9_shared_db_pool_cleanup_total{result=~"error|bad_conn|deadline_exceeded|not_cleared|not_deleted"}[15m])
          ) > 0
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 failed shared pool cleanup is not converging"
          description: "Drive9 (ns={{ $labels.namespace }}) failed shared-pool cleanup stage {{ $labels.stage }} recorded {{ $labels.result }} in the past 15 minutes. Current value: {{ $value }} failures"

      - alert: Drive9MetaDBProbeLatencyP95High
        expr: |
          (
            histogram_quantile(0.95,
              sum by (namespace, le) (rate(drive9_db_probe_duration_seconds_bucket{role="meta",result="ok"}[5m]))
            ) > 1
          )
          and
          sum by (namespace) (increase(drive9_db_probe_duration_seconds_count{role="meta",result="ok"}[5m])) > 10
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 metadata DB probe latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) metadata DB health-probe p95 exceeded 1 second for 5 minutes. This is an early control-plane latency signal before probes time out. Current value: {{ $value | humanizeDuration }}"

      # TiDB Cloud Native private endpoint mode needs either the cluster private
      # host from TiDB Cloud or a configured public_host -> private_host mapping.
      # A missing mapping leaves the tenant without a usable DB host.
      - alert: Drive9TiDBCloudPrivateEndpointHostMappingMissing
        expr: |
          (
            sum by (namespace) (
              increase(drive9_service_operations_total{component="tidbcloud_native",operation="private_endpoint_host_lookup",result="mapping_missing"}[5m])
            ) > 0
          )
          or
          (
            (
              # Keep the longer historical lookback so a temporarily stale series
              # is not mistaken for a first-seen counter when it resumes reporting.
              sum by (namespace) (
                drive9_service_operations_total{component="tidbcloud_native",operation="private_endpoint_host_lookup",result="mapping_missing"}
                unless
                max_over_time(drive9_service_operations_total{component="tidbcloud_native",operation="private_endpoint_host_lookup",result="mapping_missing"}[10m] offset 10m)
              )
            ) > 0
          )
        for: 1m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 TiDB Cloud private endpoint host mapping is missing"
          description: "Drive9 (ns={{ $labels.namespace }}) could not resolve a TiDB Cloud private endpoint host from cluster metadata or DRIVE9_TIDBCLOUD_PRIVATE_ENDPOINT_HOST_MAP. Provisioning or tenant-pool resume can leave tenants without a usable DB host. Current 5m mapping misses: {{ $value }}"

      # The code emits this only after a TiDB Cloud cluster/branch has already
      # spent 5 minutes without complete connection metadata, so keep the alert
      # rule itself short to avoid delaying the signal to 10+ minutes.
      - alert: Drive9TiDBCloudMetadataNotReadyAfter5m
        expr: |
          (
            sum by (namespace, result) (
              increase(drive9_service_operations_total{component="tidbcloud_native",operation="metadata_not_ready_5m"}[5m])
            ) > 0
          )
          or
          (
            (
              # This is a de-duplication horizon, not the alert observation window;
              # retaining it prevents recovered stale series from re-alerting.
              sum by (namespace, result) (
                drive9_service_operations_total{component="tidbcloud_native",operation="metadata_not_ready_5m"}
                unless
                max_over_time(drive9_service_operations_total{component="tidbcloud_native",operation="metadata_not_ready_5m"}[10m] offset 10m)
              )
            ) > 0
          )
        for: 1m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 TiDB Cloud metadata is not ready after 5 minutes"
          description: "Drive9 (ns={{ $labels.namespace }}) observed TiDB Cloud connection metadata still not ready after 5 minutes; reason={{ $labels.result }}. Check tidbcloud_native_*metadata_wait* logs for cluster IDs, tenant IDs, missing host/userPrefix fields, and private endpoint mapping coverage. Current 5m observations: {{ $value }}"

      # Long metadata waits usually mean TiDB Cloud clusters exist but endpoint,
      # user_prefix, or organization labels have not become visible yet. Alert by
      # pool/org so a single slow pool can be isolated without losing attribution.
      - alert: Drive9AdminTenantPoolMetadataResumeWaitLatencyHigh
        expr: |
          (
            histogram_quantile(0.95,
              sum(rate(drive9_tenant_pool_metadata_resume_wait_duration_seconds_bucket{scope="batch"}[15m])) by (namespace, pool_id, organization_id, le)
            ) > 480
          )
          and
          (
            sum by (namespace, pool_id, organization_id) (
              increase(drive9_tenant_pool_metadata_resume_wait_duration_seconds_count{scope="batch"}[30m])
            ) > 0
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 admin tenant pool metadata resume wait latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant pool {{ $labels.pool_id }} / TiDB org {{ $labels.organization_id }} metadata resume batch wait p95 exceeded 8 minutes. Check tidbcloud_native_batch_metadata_wait_* logs and group-scope metrics for the pending cluster IDs. Current value: {{ $value | humanizeDuration }}"

      # Pool free capacity is the number of active free tenants that can be
      # claimed immediately. Pending/provisioning slots are intentionally not
      # counted as free because they cannot absorb provision traffic yet.
      - alert: Drive9AdminTenantPoolFreeCapacityLow
        expr: |
          (
            max by (namespace, pool_id, organization_id) (drive9_tenant_pool_capacity{state="free"})
            /
            clamp_min(max by (namespace, pool_id, organization_id) (drive9_tenant_pool_capacity{state="size"}), 1)
          ) < 0.20
          and
          (
            max by (namespace, pool_id, organization_id) (drive9_tenant_pool_capacity{state="size"}) > 0
          )
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 admin tenant pool free capacity is low"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant pool {{ $labels.pool_id }} / TiDB org {{ $labels.organization_id }} free capacity is below 20% for 5 minutes. This ratio uses active free tenants over configured pool size; pending/provisioning slots are not counted as free. Current value: {{ $value | humanizePercentage }}"

      # Pool grow/shrink is expected to involve external TiDB Cloud calls, but
      # sustained multi-minute latency means admin set-size can hold the pool lock
      # long enough to block follow-up maintenance for that pool.
      - alert: Drive9AdminTenantPoolUpdateLatencyHigh
        expr: |
          (
            histogram_quantile(0.95,
              sum(rate(drive9_service_operation_duration_seconds_bucket{component="admin_tenant_pool",operation="update"}[5m])) by (namespace, le)
            ) > 300
          )
          and
          (
            sum by (namespace) (
              increase(drive9_service_operation_duration_seconds_count{component="admin_tenant_pool",operation="update"}[30m])
            ) > 3
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 admin tenant pool update latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) admin tenant pool update p95 exceeded 5 minutes. Check admin_tenant_pool_update_* logs for the slow grow/shrink stage and TiDB Cloud API behavior. Current value: {{ $value | humanizeDuration }}"

      # SSE persistence failures mean reconnect replay may fall back to broad reset or lose precise cache invalidation history.
      - alert: Drive9SSEPersistenceFailureRateHigh
        expr: |
          sum by (namespace) (
            increase(drive9_service_operations_total{component="event_bus",operation="publish",result=~"error|bad_conn|deadline_exceeded|.+_error"}[15m])
          ) > 0
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 SSE event persistence failure ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) failed to persist one or more durable SSE events in the past 15 minutes. Cross-pod replay may lose precise invalidations. Current value: {{ $value }} failures"

      # Replay misses force clients to reset all caches instead of applying precise durable events.
      - alert: Drive9SSEReplayMissesHigh
        expr: |
          sum by (namespace) (increase(drive9_sse_resets_sent_total{reason="seq_too_old"}[15m])) > 5
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 SSE replay misses are high"
          description: "Drive9 (ns={{ $labels.namespace }}) SSE durable replay reported more than 5 seq_too_old misses in 15 minutes. Current value: {{ $value }} misses"

      # Slow durable replay delays mount cache convergence after reconnects.
      - alert: Drive9SSEReplayP95High
        expr: |
          histogram_quantile(0.95,
            sum(rate(drive9_sse_phase1_duration_seconds_bucket[5m])) by (namespace, le)
          ) > 2
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 SSE replay p95 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) SSE durable replay p95 latency exceeded 2 seconds over 5 minutes. Current value: {{ $value | humanizeDuration }}"

      # Retention sweep failures can cause the durable event log to grow without bound.
      - alert: Drive9SSERetentionSweepFailures
        expr: |
          sum by (namespace) (
            increase(drive9_service_operations_total{component="event_bus",operation="retention_sweep",result=~"error|bad_conn|deadline_exceeded|.+_error"}[15m])
          ) > 0
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 SSE event retention sweep is failing"
          description: "Drive9 (ns={{ $labels.namespace }}) SSE event retention sweep recorded failures within the past 15 minutes. Current value: {{ $value }} failures"

      # Deployment mismatch catches the main service becoming unavailable even if app-level metrics disappear entirely.
      - alert: Drive9DeploymentReplicasMismatch
        expr: |
          (
            kube_deployment_spec_replicas{namespace=~"dat9|drive9-tidbcloud",deployment=~"dat9-server|drive9-server"}
            !=
            kube_deployment_status_replicas_available{namespace=~"dat9|drive9-tidbcloud",deployment=~"dat9-server|drive9-server"}
          ) and (
            changes(kube_deployment_status_replicas_updated{namespace=~"dat9|drive9-tidbcloud",deployment=~"dat9-server|drive9-server"}[5m]) == 0
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 deployment replicas mismatch"
          description: "Drive9 deployment {{ $labels.namespace }}/{{ $labels.deployment }} has not matched expected replicas for 5 minutes"

      # Working-set memory near the configured container limit precedes OOMKilled
      # and gives operators time to resize the pod or find a memory leak.
      - alert: Drive9PodMemoryLimitHigh
        expr: |
          (
            sum by (namespace, pod) (container_memory_working_set_bytes{namespace=~"dat9|drive9-tidbcloud",container=~"dat9-server|drive9-server"})
            /
            clamp_min(sum by (namespace, pod) (kube_pod_container_resource_limits{namespace=~"dat9|drive9-tidbcloud",container=~"dat9-server|drive9-server",resource="memory"}), 1)
          ) > 0.85
        for: 15m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 pod memory usage is high"
          description: "Drive9 (ns={{ $labels.namespace }}) pod {{ $labels.pod }} has used more than 85% of its memory limit for 15 minutes; it is at risk of OOMKilled. Current value: {{ $value | humanizePercentage }}"

      # A pending Drive9 replica may be blocked by node capacity, resource quota,
      # taints, or affinity. Deployment replica mismatch alone does not expose why.
      - alert: Drive9PodUnschedulable
        expr: |
          max by (namespace, pod) (
            kube_pod_status_scheduled{namespace=~"dat9|drive9-tidbcloud",pod=~"(dat9-server|drive9-server)-.*",condition="false"}
          ) == 1
        for: 10m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 pod is unschedulable"
          description: "Drive9 pod {{ $labels.namespace }}/{{ $labels.pod }} has remained unschedulable for 10 minutes; check node capacity, scheduling constraints, and resource requests."

      # CrashLoopBackOff is the fastest strong signal of broken rollout or fatal startup regression.
      - alert: Drive9PodCrashLooping
        expr: |
          sum by (namespace,pod,container) (
            increase(kube_pod_container_status_restarts_total{namespace=~"dat9|drive9-tidbcloud",container=~"dat9-server|drive9-server"}[5m])
          ) > 0
          and on (namespace,pod,container)
          max by (namespace,pod,container) (
            kube_pod_container_status_waiting_reason{namespace=~"dat9|drive9-tidbcloud",container=~"dat9-server|drive9-server",reason="CrashLoopBackOff"} == 1
          )
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 pod is crash looping"
          description: "Drive9 pod {{ $labels.namespace }}/{{ $labels.pod }} is in CrashLoopBackOff"

      # OOMKilled indicates the pod is unavailable and likely dropping traffic or background work.
      - alert: Drive9PodOOMKilled
        expr: |
          kube_pod_container_status_terminated_reason{namespace=~"dat9|drive9-tidbcloud",container=~"dat9-server|drive9-server",reason="OOMKilled"} == 1
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 pod was OOMKilled"
          description: "Drive9 pod {{ $labels.namespace }}/{{ $labels.pod }} was OOMKilled"

      # Vault provides secret storage and token issuance for tenants that enable it.
      # An outage hard-fails every vault-backed surface, so page it as critical even
      # though unrelated filesystem operations can remain available.
      - alert: Drive9VaultUnavailable
        expr: |
          (max by (namespace, pod) (drive9_module_up{module="vault"}) == 0)
          and on (namespace, pod)
          (max by (namespace, pod) (drive9_feature_enabled{feature="vault"}) == 1)
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 vault module is unavailable"
          description: "Drive9 (ns={{ $labels.namespace }}) vault module has been unavailable for 5 minutes while the service is still exporting metrics."

      # Catch-all error-ratio for service components that have no dedicated rule above
      # (backend upload path, quota engines, caches, datastore, meta, tenant pool, …).
      # Components that have their own dedicated rule on this same metric (sse,
      # s3client, webdav, image/audio extract, the background workers below) are
      # excluded to avoid double-paging. (FUSE has its own alerts too but reports via
      # drive9_fuse_* metrics, not drive9_service_operations_total, so it never matches
      # here and needs no exclusion.) tenant_worker is also excluded — its kick
      # "delayed" status is normal backpressure, not an error, and its error
      # surface is already covered by the semantic_worker and file_gc alerts.
      - alert: Drive9ServiceOperationErrorRateHigh
        expr: |
          (
            (
              sum by (namespace, component) (drive9_service_operations:rate5m:sum_by_namespace_component_result{result=~"error|bad_conn|.+_error"})
              /
              clamp_min(sum by (namespace, component) (drive9_service_operations:rate5m:sum_by_namespace_component_result), 0.000001)
            ) > 0.10
          )
          and
          (
            sum by (namespace, component) (drive9_service_operations:rate5m:sum_by_namespace_component_result) * 300 > 50
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 service component error ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) component {{ $labels.component }} operation errors exceeded 10% over 5 minutes. Current value: {{ $value | humanizePercentage }}"

      # Successes are aggregate-only; tenant attribution is intentionally a
      # failure-only counter, so this alert uses an absolute failure floor rather
      # than a mathematically invalid per-tenant ratio.
      - alert: Drive9ServiceOperationErrorRateHighByTenant
        expr: |
          sum by (namespace, component, tenant_id, tidbcloud_org_id) (
            increase(drive9_tenant_operation_failures_total{component!~"semantic_worker|mutation_replay|file_gc|expiry_sweep|event_bus"}[15m])
          ) > 5
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 service component failures are high for tenant"
          description: "Drive9 (ns={{ $labels.namespace }}) component {{ $labels.component }} for tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} recorded more than 5 attributable failures over 15 minutes. Current value: {{ $value }} failures"

      # Background integrity/delivery workers are low-volume, so an absolute failure count
      # is more reliable than a ratio. These back data correctness: mutation_replay drives
      # central quota/SSE/cache convergence, file_gc reclaims orphaned storage,
      # expiry_sweep clears stale uploads, and event_bus delivers change events.
      # Exclude file_gc/sched/slot_busy_error: multi-worker PerTenantConcurrency=1
      # contention is expected scheduling backpressure (tenant_worker_slot_busy), not a
      # worker hard-error. It still appears on the tenant failure series for diagnosis.
      - alert: Drive9BackgroundWorkerErrorsHigh
        expr: |
          sum by (namespace, worker, operation) (
            label_replace(
              increase(drive9_service_operations_total{component=~"mutation_replay|file_gc|expiry_sweep|event_bus",result=~"error|.+_error|deadline_exceeded",result!="slot_busy_error"}[15m]),
              "worker", "$1", "component", "(.*)"
            )
          ) > 5
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 background worker is failing"
          description: "Drive9 (ns={{ $labels.namespace }}) background worker {{ $labels.worker }}/{{ $labels.operation }} recorded more than 5 errors or deadline timeouts over 15 minutes. This includes unscoped control-plane paths such as expiry sweeps and mutation replay listing. Current value: {{ $value }} failures"

      - alert: Drive9BackgroundWorkerTenantErrorsHigh
        expr: |
          sum by (namespace, worker, operation, tenant_id, tidbcloud_org_id) (
            label_replace(
              increase(drive9_tenant_operation_failures_total{component=~"mutation_replay|file_gc|event_bus",result=~"error|.+_error|deadline_exceeded",result!="slot_busy_error"}[15m]),
              "worker", "$1", "component", "(.*)"
            )
          ) > 5
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 background worker is failing for a tenant"
          description: "Drive9 (ns={{ $labels.namespace }}) background worker {{ $labels.worker }}/{{ $labels.operation }} for tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} recorded more than 5 attributable failures over 15 minutes. Current value: {{ $value }} failures"

      # Bad DB connections are usually noisy on request paths, but sustained bad_conn in
      # correctness/convergence workers is actionable: mutation_replay can stop central
      # quota convergence, file_gc can stop reclaiming storage, and event delivery can
      # stall. Keep this separate from Drive9BackgroundWorkerErrorsHigh so transient
      # reconnect churn does not pollute the generic worker error signal. The major
      # threshold is loose enough to avoid paging on one-off reconnect churn.
      - alert: Drive9BackgroundWorkerBadConnHigh
        expr: |
          sum by (namespace, worker, operation) (
            label_replace(
              increase(drive9_service_operations_total{component=~"mutation_replay|file_gc|expiry_sweep|event_bus",result="bad_conn"}[15m]),
              "worker", "$1", "component", "(.*)"
            )
          ) > 10
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 background worker has sustained bad DB connections"
          description: "Drive9 (ns={{ $labels.namespace }}) background worker {{ $labels.worker }}/{{ $labels.operation }} recorded more than 10 bad DB connections over 15 minutes. For mutation_replay this can slow central quota convergence and leave quota counters stale. Current value: {{ $value }} bad connections"

      - alert: Drive9BackgroundWorkerBadConnCritical
        expr: |
          sum by (namespace, worker, operation) (
            label_replace(
              increase(drive9_service_operations_total{component=~"mutation_replay|file_gc|expiry_sweep|event_bus",result="bad_conn"}[15m]),
              "worker", "$1", "component", "(.*)"
            )
          ) > 30
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 background worker has severe bad DB connections"
          description: "Drive9 (ns={{ $labels.namespace }}) background worker {{ $labels.worker }}/{{ $labels.operation }} recorded more than 30 bad DB connections over 15 minutes. For mutation_replay this can stop central quota convergence and leave quota counters stale. Current value: {{ $value }} bad connections"

      # Meta mutation log insert is the durable quota handoff after tenant file
      # writes commit. Any log_error means the request surfaced an error, but the
      # tenant write may already exist and require operational quota reconciliation.
      - alert: Drive9CentralQuotaMutationLogInsertFailures
        expr: |
          sum by (namespace, tenant_id, tidbcloud_org_id, operation) (
            increase(drive9_tenant_operation_failures_total{component="central_quota",result="log_error"}[15m])
          ) > 0
        for: 0m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 central quota mutation log insert is failing"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} central quota operation {{ $labels.operation }} failed to insert quota_mutation_log entries within the past 15 minutes. The tenant write may already have committed, so central quota can be stale until replay or operational reconciliation. Current value: {{ $value }} log insert failures"

      # mutation_replay is the primary central quota convergence worker after
      # tenant-local outbox accounting was removed from the runtime hot path.
      - alert: Drive9QuotaMutationReplayFailures
        expr: |
          sum by (namespace, tenant_id, tidbcloud_org_id, operation) (
            increase(drive9_tenant_operation_failures_total{component="mutation_replay",result=~"error|bad_conn|.+_error|deadline_exceeded"}[15m])
          ) > 3
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 quota mutation replay is failing"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} quota mutation replay for {{ $labels.operation }} recorded more than 3 failures over 15 minutes. This can delay central quota convergence and leave quota counters stale. Current value: {{ $value }} failures"

      - alert: Drive9QuotaMutationReplayBacklogHigh
        expr: |
          max by (namespace, tenant_id, tidbcloud_org_id) (
            drive9_service_gauge{component="mutation_replay",name="pending_mutations",tenant_id!="",tenant_id!="unknown"}
          ) > 1000
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 quota mutation replay backlog is high"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} has more than 1000 pending quota mutation log entries for 15 minutes. This can delay central quota convergence and leave quota counters stale. Current value: {{ $value }} pending mutations"

      - alert: Drive9QuotaMutationReplayOldestPendingAgeHigh
        expr: |
          max by (namespace, tenant_id, tidbcloud_org_id) (
            drive9_service_gauge{component="mutation_replay",name="oldest_pending_age_seconds",tenant_id!="",tenant_id!="unknown"}
          ) > 300
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 quota mutation replay pending age is high"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} has quota mutation log entries pending for more than 5 minutes. This can delay central quota convergence and leave quota counters stale. Current value: {{ $value }}s"

      # Quota fail-open is exported through the failure-only tenant counter.
      # Successes remain aggregate-only, so use an absolute per-tenant count.
      - alert: Drive9QuotaCheckFailOpenRateHigh
        expr: |
          sum by (namespace, tenant_id, tidbcloud_org_id) (
            increase(drive9_tenant_operation_failures_total{component=~"server_quota|central_quota",result=~"fail_open|busy_fail_open"}[15m])
          ) > 10
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 quota checks are repeatedly failing open"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} quota engine recorded more than 10 fail-open admissions over 15 minutes. Operations are being admitted without complete enforcement. Current value: {{ $value }} fail-open admissions"

      # Goroutine leak (Saturation golden signal). Threshold is intentionally conservative;
      # tune per observed steady-state. Sustained unbounded growth precedes OOM / scheduler
      # starvation. go_goroutines is exported by the in-process Go runtime collector.
      - alert: Drive9GoroutineLeak
        expr: |
          go_goroutines{namespace=~"dat9|drive9-tidbcloud"} > 10000
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 goroutine count is high"
          description: "Drive9 (ns={{ $labels.namespace }}) process {{ $labels.pod }} has held more than 10000 goroutines for 15 minutes, indicating a likely goroutine leak. Current value: {{ $value }}"

      # GC pressure (Saturation golden signal): more than 10% of wall-clock time spent in
      # stop-the-world GC pauses points to heap churn / undersized memory before OOM appears.
      - alert: Drive9HighGCCPUFraction
        expr: |
          rate(go_gc_pause_seconds_total{namespace=~"dat9|drive9-tidbcloud"}[5m]) > 0.10
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 GC pause time is high"
          description: "Drive9 (ns={{ $labels.namespace }}) process {{ $labels.pod }} has spent more than 10% of wall-clock time in GC pauses for 15 minutes. Current value: {{ $value | humanizePercentage }}"

      # No pod holds leadership across the fleet: all leader-gated background workers
      # (mutation_replay, expiry_sweep, object_gc, safety_net_scan) have
      # stalled. Semantic task processing and file_gc are no longer leader-gated —
      # they run on every pod via the unified tenant outbox poller + kick model.
      # The HTTP serving path can remain available, but every leader-gated control-plane
      # worker is stalled, so treat this as a critical control-plane outage. A brief
      # no-leader window during failover is expected; the 5m hold absorbs it. The gauge
      # is only exported once a pod has reported, so this never fires before election starts.
      - alert: Drive9NoLeaderElected
        expr: |
          max by (namespace) (drive9_service_gauge{component="leader",name="is_leader"}) == 0
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 has no elected leader"
          description: "No Drive9 pod has held leader election (ns={{ $labels.namespace }}) for 5 minutes; all leader-gated background workers are stalled. Note: semantic task processing and file_gc are not affected — they run on every pod via the unified tenant outbox."

      # Embedding/LLM provider is a hard external dependency for semantic search and
      # async extraction. Sustained outbound embedding errors point to provider outage,
      # auth, or rate-limit problems before they surface as semantic dead-lettering.
      - alert: Drive9EmbeddingProviderErrorRateHigh
        expr: |
          (
            (
              sum by (namespace) (rate(drive9_service_operations_total{component="embedding",operation="embed_text",result!="ok"}[5m]))
              /
              clamp_min(sum by (namespace) (rate(drive9_service_operations_total{component="embedding",operation="embed_text"}[5m])), 0.000001)
            ) > 0.20
          )
          and
          (
            sum by (namespace) (increase(drive9_service_operations_total{component="embedding",operation="embed_text"}[5m])) > 20
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 embedding provider error ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) outbound embedding calls exceeded 20% errors over 5 minutes (provider outage, auth, or rate limiting). Current value: {{ $value | humanizePercentage }}"

      # KMS/encryption failures are rare but high-impact and low-volume, so an absolute
      # count is more reliable than a ratio: a few decrypt failures mean a tenant's DB
      # password cannot be unsealed and that tenant becomes unusable.
      - alert: Drive9EncryptionFailures
        expr: |
          sum by (namespace, operation) (
            increase(drive9_service_operations_total{component="encrypt",result="error"}[15m])
          ) > 5
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 encryption/KMS operations are failing"
          description: "Drive9 (ns={{ $labels.namespace }}) recorded more than 5 encrypt/decrypt ({{ $labels.operation }}) failures over 15 minutes; KMS may be unreachable or a key disabled. Current value: {{ $value }} failures"

      # --- Performance / saturation signals -------------------------------------

      # File-descriptor exhaustion (Saturation): once the process runs out of fds it
      # can no longer accept connections or open sockets/files -> directly impacts
      # availability, hence critical. process_open_fds / process_max_fds are exported
      # by the in-process /proc collector.
      - alert: Drive9FileDescriptorsCritical
        expr: |
          (
            process_open_fds{namespace=~"dat9|drive9-tidbcloud"}
            /
            clamp_min(process_max_fds{namespace=~"dat9|drive9-tidbcloud"}, 1)
          ) > 0.95
        for: 5m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 is nearly out of file descriptors"
          description: "Drive9 (ns={{ $labels.namespace }}) process {{ $labels.pod }} has used more than 95% of its file-descriptor limit for 5 minutes; new connections will start failing. Current value: {{ $value | humanizePercentage }}"

      # Early file-descriptor pressure: actionable before exhaustion (leak or undersized
      # ulimit), but not yet impacting traffic -> major.
      - alert: Drive9FileDescriptorsHigh
        expr: |
          (
            process_open_fds{namespace=~"dat9|drive9-tidbcloud"}
            /
            clamp_min(process_max_fds{namespace=~"dat9|drive9-tidbcloud"}, 1)
          ) > 0.85
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 file-descriptor usage is high"
          description: "Drive9 (ns={{ $labels.namespace }}) process {{ $labels.pod }} has used more than 85% of its file-descriptor limit for 15 minutes. Current value: {{ $value | humanizePercentage }}"

      # CPU saturation via CFS throttling (Saturation): sustained throttling means the
      # container is CPU-starved and request latency will suffer. Uses kube container
      # metrics, scoped to the drive9 deployment like the other k8s-level rules.
      - alert: Drive9CPUThrottlingHigh
        expr: |
          (
            sum by (namespace, pod) (rate(container_cpu_cfs_throttled_periods_total{namespace=~"dat9|drive9-tidbcloud",container=~"dat9-server|drive9-server"}[5m]))
            /
            clamp_min(sum by (namespace, pod) (rate(container_cpu_cfs_periods_total{namespace=~"dat9|drive9-tidbcloud",container=~"dat9-server|drive9-server"}[5m])), 0.000001)
          ) > 0.25
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 is being CPU throttled"
          description: "Drive9 (ns={{ $labels.namespace }}) pod {{ $labels.pod }} has been CPU-throttled on more than 25% of CFS periods for 15 minutes; consider raising CPU limits. Current value: {{ $value | humanizePercentage }}"

      # Core upload-path latency (Latency): confirm/finalize/patch are the user-visible
      # write operations. They have a duration histogram but no latency alert until now.
      - alert: Drive9BackendUploadLatencyP95High
        expr: |
          histogram_quantile(0.95,
            sum by (namespace, le, operation) (rate(drive9_service_operation_duration_seconds_bucket{component="backend",operation=~"confirm_upload|finalize_upload|patch_upload"}[5m]))
          ) > 15
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 upload-path p95 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) backend upload operation {{ $labels.operation }} p95 latency exceeded 15 seconds over 5 minutes. Current value: {{ $value | humanizeDuration }}"

      # Embedding/LLM provider latency (Latency): slow embeddings stall semantic search
      # and async extraction even when the provider is not outright erroring.
      - alert: Drive9EmbeddingProviderLatencyP99High
        expr: |
          histogram_quantile(0.99,
            sum by (namespace, le) (rate(drive9_service_operation_duration_seconds_bucket{component="embedding",operation="embed_text"}[5m]))
          ) > 5
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 embedding provider p99 latency is high"
          description: "Drive9 (ns={{ $labels.namespace }}) outbound embedding call p99 latency exceeded 5 seconds over 5 minutes. Current value: {{ $value | humanizeDuration }}"

      # --- User DB access / scan detection -----------------------------------
      # No periodic goroutine may full-scan tenant TiDBs: scans wake serverless
      # instances and cause billing storms. The alerts below guard that
      # invariant. They use a single metric family:
      #   drive9_service_operations_total{component="user_db_access",operation=<path>,result=...}
      # Each operation label identifies the calling path that touched a tenant
      # user DB. The key signals are:
      #   acquire_cold_open — a tenant backend was created from scratch.
      #     Caveat: this operation does not distinguish tenant schema shapes.
      #     For standalone tenants a cold open wakes a dedicated serverless
      #     TiDB (the canonical billing-storm indicator). For shared-schema
      #     tenants (tidb_cloud_native_shared) the backend is created through
      #     an already-warm shared DB handle, so a cold open is only a
      #     per-pod LRU cache miss — no serverless instance is woken. As the
      #     fleet migrates to shared schema, the cold-open rate is therefore
      #     a churn signal, not a pure billing signal. Only request traffic
      #     and kicks may produce cold opens, never periodic scans.
      #   acquire_cached — warm-only safety-net access (hit / cold_skipped / stale_skipped).
      #   safety_net_* — the periodic 5min warm-only recovery scan.
      #   object_gc_acquire — leader-gated 1min object GC (opens cold tenants).
      #   tenant_worker_acquire — kick-driven worker acquire (follows writes).
      #   auth_acquire — request-path acquire (the expected hot-path baseline).
      #   outbox_dispatch_kick — poller dispatched a kick that will touch a tenant DB.

      # Safety-net scan list failures. The scan is the last-resort recovery for
      # lost kicks; repeated meta-DB list failures mean warm-tenant recovery is
      # degraded. Warning because the primary delivery path (outbox poller) still
      # runs — this only degrades the safety net.
      - alert: Drive9SafetyNetScanFailureHigh
        expr: |
          sum by (namespace) (increase(drive9_service_operations_total{component="user_db_access",operation="safety_net_scan_list",result="error"}[15m])) > 3
        for: 5m
        labels:
          severity: warning
          response_class: logging
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 safety-net scan list failures are high"
          description: "Drive9 (ns={{ $labels.namespace }}) safety-net scan recorded more than 3 meta-DB list failures over 15 minutes. Warm-tenant expired-lease recovery is degraded; the outbox poller remains the primary delivery path."

      # Safety-net scanning too many warm tenants per cycle. A sudden spike
      # suggests the warm cache grew unexpectedly large or the scan loop is
      # iterating more than expected — a precursor to a scan touching many DBs.
      # The safety-net is warm-only by design, so this should track the warm
      # cache size, not the total active-tenant count.
      - alert: Drive9SafetyNetScanTenantScanRateHigh
        expr: |
          sum by (namespace) (rate(drive9_service_operations_total{component="user_db_access",operation="safety_net_tenant_scan"}[5m])) > 50
        for: 5m
        labels:
          severity: warning
          response_class: logging
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 safety-net tenant scan rate is high"
          description: "Drive9 (ns={{ $labels.namespace }}) safety-net scan is touching more than 50 warm tenants/sec over 5 minutes. A spike suggests the warm cache grew unexpectedly or the scan is iterating beyond its intended scope — investigate before it becomes a billing-storm contributor."

      # Object-GC is leader-gated and opens cold tenants. A spike means many
      # tenant TiDBs are being woken for GC — a potential billing-storm
      # contributor if a bug causes repeated candidate reprocessing.
      - alert: Drive9ObjectGCAcquireRateHigh
        expr: |
          sum by (namespace) (rate(drive9_service_operations_total{component="user_db_access",operation="object_gc_acquire"}[5m])) > 5
        for: 5m
        labels:
          severity: warning
          response_class: logging
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 object GC tenant acquire rate is high"
          description: "Drive9 (ns={{ $labels.namespace }}) object GC is acquiring tenant backends at more than 5/sec over 5 minutes. Object-GC opens cold tenant TiDBs; a sustained spike may wake many serverless instances and incur cost. Check for a bug causing repeated candidate reprocessing."

      # Sustained cold-open churn across the fleet. Originally the primary
      # billing-storm signal (cold opens = TiDB wake-ups), but the metric does
      # not distinguish schema shapes: shared-schema tenants cold-open through
      # an already-warm shared DB handle, so their cold opens are ordinary
      # per-pod LRU cache misses, not serverless wake-ups. The rate is
      # therefore reframed as a churn signal: sustained elevation means the
      # per-pod LRU is undersized, a rollout is churning backends, or a
      # periodic scan path is waking idle tenant DBs. Downgraded to warning
      # with a 20/s / 15m threshold (~18,000 backend creations) so normal
      # shared tenant churn does not page; the acute billing signature stays covered
      # by Drive9UserDBAcquireColdOpenBurst, and the warm-only invariant
      # itself is guarded by Drive9SafetyNetScanColdOpenInvariantViolated and
      # Drive9UserDBAccessUnexpectedPath.
      - alert: Drive9UserDBAcquireColdOpenRateHigh
        expr: |
          sum by (namespace) (rate(drive9_service_operations_total{component="user_db_access",operation="acquire_cold_open",result="ok"}[15m])) > 20
        for: 15m
        labels:
          severity: warning
          response_class: logging
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 user DB cold-open churn is high"
          description: "Drive9 (ns={{ $labels.namespace }}) cold-open rate has stayed above 20 tenant backend cold opens/sec (15-minute rate window) for a 15-minute sustain period. Shared-schema cold opens are normal per-pod LRU cache misses and do not wake serverless TiDBs, so this is a churn signal: check whether the per-pod LRU is undersized, a rollout is churning backends, or a periodic scan path is waking idle tenant DBs. Current value: {{ $value }} cold opens/sec"

      # Acute cold-open burst. For standalone tenants this is still the exact
      # billing-storm signature — a scan or bug rapidly waking many serverless
      # TiDBs. For shared-schema tenants the same burst means pathological
      # backend churn (mass cache invalidation or a scan loop) that can
      # exhaust shared DB pool connections. A normal restart or rollout
      # legitimately produces on the order of a thousand cold opens in 2
      # minutes as fresh pods warm their per-pod LRU caches, so the threshold
      # sits well above that: more than 5000 cold opens in 2 minutes is never
      # normal traffic under either schema shape. It also guards the
      # guarantee that no periodic goroutine touches idle tenant DBs.
      - alert: Drive9UserDBAcquireColdOpenBurst
        expr: |
          sum by (namespace) (increase(drive9_service_operations_total{component="user_db_access",operation="acquire_cold_open",result="ok"}[2m])) > 5000
        for: 1m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 user DB cold-open burst detected"
          description: "Drive9 (ns={{ $labels.namespace }}) opened more than 5000 tenant backends from scratch (cold opens) in 2 minutes — far above the ~1000 a normal restart or rollout produces. For standalone tenants this is the billing-storm signature — a scan or bug is rapidly waking serverless TiDBs; for shared-schema tenants it indicates pathological backend churn that can exhaust shared DB pool connections. Investigate immediately: check for a new periodic scan path. Current value: {{ $value }} cold opens in 2m"

      # The invariant guard. AcquireCached must never open a cold tenant TiDB —
      # it returns cold_skipped on cache miss. If it ever records an error or the
      # safety-net starts producing cold opens at the scan-cycle rate, the
      # warm-only invariant is broken — the guarantee that no periodic
      # goroutine ever touches idle tenant TiDBs no longer holds.
      - alert: Drive9SafetyNetScanColdOpenInvariantViolated
        expr: |
          sum by (namespace) (increase(drive9_service_operations_total{component="user_db_access",operation="acquire_cached",result=~"error|cold_open"}[5m])) > 0
        for: 1m
        labels:
          severity: critical
          response_class: page
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 safety-net warm-only invariant violated"
          description: "Drive9 (ns={{ $labels.namespace }}) AcquireCached recorded an error or cold-open result within 5 minutes. AcquireCached must never open a cold tenant TiDB — it should only return hit/cold_skipped/stale_skipped. This violates the invariant that no periodic goroutine touches idle tenant DBs. Investigate immediately."

      # Kick-driven tenant-worker acquire errors. Sustained errors mean kicks
      # are failing to reach the tenant DB — semantic/file_gc work is not being
      # processed. Major because it degrades async processing, not request availability.
      - alert: Drive9TenantWorkerAcquireErrorRateHigh
        expr: |
          (
            (
              sum by (namespace) (rate(drive9_service_operations_total{component="user_db_access",operation="tenant_worker_acquire",result=~"error|bad_conn"}[5m]))
              /
              clamp_min(sum by (namespace) (rate(drive9_service_operations_total{component="user_db_access",operation="tenant_worker_acquire"}[5m])), 0.000001)
            ) > 0.10
          )
          and
          (
            sum by (namespace) (increase(drive9_service_operations_total{component="user_db_access",operation="tenant_worker_acquire"}[5m])) > 20
          )
        for: 5m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 tenant worker acquire error ratio is high"
          description: "Drive9 (ns={{ $labels.namespace }}) kick-driven tenant-worker acquire errors exceeded 10% over 5 minutes. Kicks are failing to reach the tenant DB — semantic and file_gc work is not being processed. Current value: {{ $value | humanizePercentage }}"

      # Catches any user-DB access from a path/operation NOT in the known
      # allowlist. If a new code path starts touching tenant DBs (a future
      # periodic scan regression), this fires. The allowlist covers the
      # instrumented operations added alongside this alert. Warning because a new
      # path may be intentional, but it must be reviewed to confirm it is not a
      # periodic scan that wakes idle tenant TiDBs.
      - alert: Drive9UserDBAccessUnexpectedPath
        expr: |
          sum by (namespace, operation) (rate(drive9_service_operations_total{component="user_db_access",operation!~"auth_acquire|tenant_worker_acquire|acquire_cold_open|acquire_cached|safety_net_scan_cycle|safety_net_tenant_scan|safety_net_scan_list|safety_net_semantic_recover|safety_net_file_gc_recover|object_gc_acquire|object_gc_list|outbox_dispatch_kick|outbox_poll_list"}[5m])) > 0.5
        for: 5m
        labels:
          severity: warning
          response_class: logging
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 user DB access from unexpected path"
          description: "Drive9 (ns={{ $labels.namespace }}) operation {{ $labels.operation }} is accessing tenant user DBs but is not in the known allowlist. A new code path may be touching tenant TiDBs — review it to confirm it is not a periodic scan that would wake idle tenant TiDBs. Current value: {{ $value }} ops/sec"

      # Tenant-level File GC reclaim lag: require a non-zero claimable queue AND
      # oldest claimable age > 1h for 15m. Gauges are sparse (series omitted at
      # zero) and only emitted for already warm/observed tenant backends on a
      # kick/drain path — not a cold-tenant fleet scan. Use max by (tenant)
      # because multiple pods may observe the same tenant without POD_ID sharding.
      # Suppress tenants whose last drain filled the batch (drive9_file_gc_drain_hit_cap):
      # they are still making progress (cold-wake stock being reclaimed).
      - alert: Drive9FileGCClaimableBacklogStale
        # Put age on the left of `and` so $value is oldest age (not claimable depth).
        expr: |
          (
            max by (namespace, tenant_id, tidbcloud_org_id) (
              drive9_file_gc_oldest_claimable_age_seconds{tenant_id!="",tenant_id!="unknown"}
            ) > 3600
          )
          and on (namespace, tenant_id, tidbcloud_org_id)
          (
            max by (namespace, tenant_id, tidbcloud_org_id) (
              drive9_file_gc_claimable{tenant_id!="",tenant_id!="unknown"}
            ) > 0
          )
          unless on (namespace, tenant_id, tidbcloud_org_id)
          (
            max by (namespace, tenant_id, tidbcloud_org_id) (
              drive9_file_gc_drain_hit_cap{tenant_id!="",tenant_id!="unknown"}
            ) > 0
          )
        for: 15m
        labels:
          severity: major
          response_class: ticket
          component: drive9
          oncall_team: drive9
        annotations:
          summary: "Drive9 file GC claimable backlog is stale for a tenant"
          description: "Drive9 (ns={{ $labels.namespace }}) tenant {{ $labels.tenant_id }} / TiDB org {{ $labels.tidbcloud_org_id }} has claimable file_gc tasks (claimable > 0) whose oldest available_at is older than 1 hour for 15 minutes. Storage quota may stay inflated after deletes. This signal is emitted only for already warm/observed tenant backends on a kick/drain path, not a cold-tenant fleet scan. Check tenant_worker_slot_busy / slot_held_long / kick_delayed logs and file_gc_tasks. Current oldest age: {{ $value | humanizeDuration }}"