rule_files: - ../rules/mem9/mnemos/drive9-alerts.yaml evaluation_interval: 1m tests: - interval: 1m input_series: # Sustained alerting failures. The unrelated high-volume list_clusters # successes prove grouping by api+operation does not dilute create_cluster. - series: 'drive9_tidbcloud_openapi_requests_total{namespace="mem9",api="cluster",operation="create_cluster",result="upstream_error",pod="drive9-0"}' values: '0+1x12' - series: 'drive9_tidbcloud_openapi_requests_total{namespace="mem9",api="cluster",operation="create_cluster",result="ok",pod="drive9-0"}' values: '0+2x12' - series: 'drive9_tidbcloud_openapi_requests_total{namespace="mem9",api="cluster",operation="list_clusters",result="ok",pod="drive9-0"}' values: '0+100x12' - series: 'drive9_tidbcloud_openapi_requests_total{namespace="mem9",api="iam",operation="resolve_api_key_identity",result="transport_error",pod="drive9-0"}' values: '0+1x12' - series: 'drive9_tidbcloud_openapi_requests_total{namespace="mem9",api="billing",operation="get_organization_plan",result="rate_limited",pod="drive9-0"}' values: '0+1x12' # Visible but deliberately non-alerting outcomes. - series: 'drive9_tidbcloud_openapi_requests_total{namespace="mem9",api="iam",operation="client_error_only",result="client_error",pod="drive9-0"}' values: '0+2x12' - series: 'drive9_tidbcloud_openapi_requests_total{namespace="mem9",api="cluster",operation="caller_canceled",result="canceled",pod="drive9-0"}' values: '0+2x12' # Alerting ratio is 100%, but fewer than three errors occur in the window. - series: 'drive9_tidbcloud_openapi_requests_total{namespace="mem9",api="billing",operation="below_error_floor",result="upstream_error",pod="drive9-0"}' values: '0 0 0 1 1 1 2 2 2 2 2 2 2' # Exactly 20% is below the strict >20% threshold despite enough errors. - series: 'drive9_tidbcloud_openapi_requests_total{namespace="mem9",api="cluster",operation="at_ratio_boundary",result="upstream_error",pod="drive9-0"}' values: '0+1x12' - series: 'drive9_tidbcloud_openapi_requests_total{namespace="mem9",api="cluster",operation="at_ratio_boundary",result="ok",pod="drive9-0"}' values: '0+4x12' alert_rule_test: - eval_time: 10m alertname: Drive9TiDBCloudOpenAPIErrorRateHigh exp_alerts: - exp_labels: namespace: mem9 api: cluster operation: create_cluster severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 TiDB Cloud OpenAPI error ratio is high" description: "Drive9 (ns=mem9) TiDB Cloud cluster/create_cluster alerting error ratio exceeded 20% over 5 minutes with at least 3 errors. Current value: 33.33%" - exp_labels: namespace: mem9 api: iam operation: resolve_api_key_identity severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 TiDB Cloud OpenAPI error ratio is high" description: "Drive9 (ns=mem9) TiDB Cloud iam/resolve_api_key_identity alerting error ratio exceeded 20% over 5 minutes with at least 3 errors. Current value: 100%" - exp_labels: namespace: mem9 api: billing operation: get_organization_plan severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 TiDB Cloud OpenAPI error ratio is high" description: "Drive9 (ns=mem9) TiDB Cloud billing/get_organization_plan alerting error ratio exceeded 20% over 5 minutes with at least 3 errors. Current value: 100%" - interval: 1m input_series: - series: 'drive9_service_operations_total{namespace="mem9",component="tidbcloud_native",operation="private_endpoint_host_lookup",result="mapping_missing",pod="drive9-0"}' values: '_x10 1 1' - series: 'drive9_service_operations_total{namespace="mem9",component="tidbcloud_native",operation="metadata_not_ready_5m",result="public_host_missing",pod="drive9-0"}' values: '_x10 1 1' alert_rule_test: - eval_time: 11m alertname: Drive9TiDBCloudPrivateEndpointHostMappingMissing exp_alerts: - exp_labels: namespace: mem9 severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 TiDB Cloud private endpoint host mapping is missing" description: "Drive9 (ns=mem9) could not resolve a TiDB Cloud private endpoint host from cluster metadata or DRIVE9_TIDBCLOUD_PRIVATE_ENDPOINT_HOST_MAP. Provisioning or tenant-pool resume can leave tenants without a usable DB host. Current 5m mapping misses: 1" - eval_time: 11m alertname: Drive9TiDBCloudMetadataNotReadyAfter5m exp_alerts: - exp_labels: namespace: mem9 result: public_host_missing severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 TiDB Cloud metadata is not ready after 5 minutes" description: "Drive9 (ns=mem9) observed TiDB Cloud connection metadata still not ready after 5 minutes; reason=public_host_missing. Check tidbcloud_native_*metadata_wait* logs for cluster IDs, tenant IDs, missing host/userPrefix fields, and private endpoint mapping coverage. Current 5m observations: 1" - interval: 1m input_series: - series: 'drive9_service_operations_total{namespace="mem9",component="tidbcloud_native",operation="private_endpoint_host_lookup",result="mapping_missing",pod="drive9-old"}' values: '1+0x12' - series: 'drive9_service_operations_total{namespace="mem9",component="tidbcloud_native",operation="private_endpoint_host_lookup",result="mapping_missing",pod="drive9-new"}' values: '_x10 1 1' - series: 'drive9_service_operations_total{namespace="mem9",component="tidbcloud_native",operation="metadata_not_ready_5m",result="public_host_missing",pod="drive9-old"}' values: '1+0x12' - series: 'drive9_service_operations_total{namespace="mem9",component="tidbcloud_native",operation="metadata_not_ready_5m",result="public_host_missing",pod="drive9-new"}' values: '_x10 1 1' alert_rule_test: - eval_time: 11m alertname: Drive9TiDBCloudPrivateEndpointHostMappingMissing exp_alerts: - exp_labels: namespace: mem9 severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 TiDB Cloud private endpoint host mapping is missing" description: "Drive9 (ns=mem9) could not resolve a TiDB Cloud private endpoint host from cluster metadata or DRIVE9_TIDBCLOUD_PRIVATE_ENDPOINT_HOST_MAP. Provisioning or tenant-pool resume can leave tenants without a usable DB host. Current 5m mapping misses: 1" - eval_time: 11m alertname: Drive9TiDBCloudMetadataNotReadyAfter5m exp_alerts: - exp_labels: namespace: mem9 result: public_host_missing severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 TiDB Cloud metadata is not ready after 5 minutes" description: "Drive9 (ns=mem9) observed TiDB Cloud connection metadata still not ready after 5 minutes; reason=public_host_missing. Check tidbcloud_native_*metadata_wait* logs for cluster IDs, tenant IDs, missing host/userPrefix fields, and private endpoint mapping coverage. Current 5m observations: 1" - interval: 1m input_series: - series: 'drive9_service_operations_total{namespace="mem9",component="tidbcloud_native",operation="private_endpoint_host_lookup",result="mapping_missing",pod="drive9-flap"}' values: '1+0x3 _x7 1+0x8' - series: 'drive9_service_operations_total{namespace="mem9",component="tidbcloud_native",operation="metadata_not_ready_5m",result="public_host_missing",pod="drive9-flap"}' values: '1+0x3 _x7 1+0x8' alert_rule_test: - eval_time: 15m alertname: Drive9TiDBCloudPrivateEndpointHostMappingMissing exp_alerts: [] - eval_time: 15m alertname: Drive9TiDBCloudMetadataNotReadyAfter5m exp_alerts: [] - interval: 1m input_series: - series: 'drive9_service_gauge{namespace="mem9",pod="drive9-critical",component="tenant_pool",name="cached_backends",tenant_id="",tidbcloud_org_id="guest"}' values: '81+0x5' - series: 'drive9_service_gauge{namespace="mem9",pod="drive9-critical",component="tenant_pool",name="max_backends",tenant_id="",tidbcloud_org_id="guest"}' values: '100+0x5' - series: 'drive9_service_gauge{namespace="mem9",pod="drive9-at-threshold",component="tenant_pool",name="cached_backends",tenant_id="",tidbcloud_org_id="guest"}' values: '80+0x5' - series: 'drive9_service_gauge{namespace="mem9",pod="drive9-at-threshold",component="tenant_pool",name="max_backends",tenant_id="",tidbcloud_org_id="guest"}' values: '100+0x5' - series: 'drive9_service_gauge{namespace="mem9",pod="drive9-transient",component="tenant_pool",name="cached_backends",tenant_id="",tidbcloud_org_id="guest"}' values: '81 81 79 79 79 79' - series: 'drive9_service_gauge{namespace="mem9",pod="drive9-transient",component="tenant_pool",name="max_backends",tenant_id="",tidbcloud_org_id="guest"}' values: '100+0x5' alert_rule_test: - eval_time: 2m alertname: Drive9UserDBLRUCapacityCritical exp_alerts: [] - eval_time: 4m alertname: Drive9UserDBLRUCapacityCritical exp_alerts: - exp_labels: namespace: mem9 pod: drive9-critical severity: critical response_class: page component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 user DB LRU cache is near capacity" description: "Drive9 (ns=mem9) pod drive9-critical user DB LRU cache has exceeded 80% of its configured capacity for 3 minutes. Current usage: 81%" - interval: 1m input_series: - series: 'drive9_tenant_requests_total{namespace="mem9",tenant_id="tenant-97",tidbcloud_org_id="97",surface="fs",status_class="5xx"}' values: '0+10x12' - series: 'drive9_tenant_requests_total{namespace="mem9",tenant_id="tenant-97",tidbcloud_org_id="97",surface="fs",status_class="2xx"}' values: '0+0x12' - series: 'drive9_tenant_requests_total{namespace="mem9",tenant_id="tenant-98",tidbcloud_org_id="98",surface="fs",status_class="5xx"}' values: '0+10x12' - series: 'drive9_tenant_requests_total{namespace="mem9",tenant_id="tenant-98",tidbcloud_org_id="98",surface="fs",status_class="2xx"}' values: '0+0x12' # Successful control-plane requests are aggregate-only in Drive9. A # tenant-attributed control-plane 5xx must not be evaluated as a # per-tenant ratio without its matching success denominator. - series: 'drive9_tenant_requests_total{namespace="mem9",tenant_id="tenant-control",tidbcloud_org_id="control",surface="provision",status_class="5xx"}' values: '0+10x12' alert_rule_test: - eval_time: 10m alertname: Drive9TenantServerErrorRateHigh exp_alerts: - exp_labels: namespace: mem9 tenant_id: tenant-97 tidbcloud_org_id: "97" surface: fs severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 tenant server error ratio is high" description: "Drive9 (ns=mem9) tenant tenant-97 / TiDB org 97 surface fs exceeded 10% 5xx responses over 5 minutes. Current value: 100%" - exp_labels: namespace: mem9 tenant_id: tenant-98 tidbcloud_org_id: "98" surface: fs severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 tenant server error ratio is high" description: "Drive9 (ns=mem9) tenant tenant-98 / TiDB org 98 surface fs exceeded 10% 5xx responses over 5 minutes. Current value: 100%" - interval: 1m input_series: - series: 'drive9_service_operations_total{namespace="mem9",component="backend",operation="write",result="error"}' values: '0+20x12' - series: 'drive9_service_operations_total{namespace="mem9",component="backend",operation="write",result="ok"}' values: '0+0x12' - series: 'drive9_tenant_operation_failures_total{namespace="mem9",component="backend",operation="write",result="error",tenant_id="tenant-97",tidbcloud_org_id="97"}' values: '0+20x12' alert_rule_test: - eval_time: 10m alertname: Drive9ServiceOperationErrorRateHigh exp_alerts: - exp_labels: namespace: mem9 component: drive9 severity: major response_class: ticket oncall_team: drive9 exp_annotations: summary: "Drive9 service component error ratio is high" description: "Drive9 (ns=mem9) component backend operation errors exceeded 10% over 5 minutes. Current value: 100%" - eval_time: 10m alertname: Drive9ServiceOperationErrorRateHighByTenant exp_alerts: - exp_labels: namespace: mem9 component: drive9 tenant_id: tenant-97 tidbcloud_org_id: "97" severity: major response_class: ticket oncall_team: drive9 exp_annotations: summary: "Drive9 service component failures are high for tenant" description: "Drive9 (ns=mem9) component backend for tenant tenant-97 / TiDB org 97 recorded more than 5 attributable failures over 15 minutes. Current value: 200 failures" - interval: 1m input_series: - series: 'drive9_db_pool_wait_count_total{namespace="mem9",role="shared",db_pool_uuid="pool-hot",tidbcloud_org_id="97",pod="drive9-0"}' values: '0+36x12 432+0x8' - series: 'drive9_db_pool_wait_count_total{namespace="mem9",role="shared",db_pool_uuid="pool-hot",tidbcloud_org_id="97",pod="drive9-1"}' values: '0+36x12 432+0x8' - series: 'drive9_db_pool_wait_count_total{namespace="mem9",role="shared",db_pool_uuid="pool-cool",tidbcloud_org_id="98",pod="drive9-0"}' values: '0+30x12' - series: 'drive9_shared_db_pool_total{namespace="mem9",db_pool_id="1",db_pool_uuid="pool-failed",tidbcloud_org_id="97",status="failed",pod="drive9-0"}' values: '1+0x12' - series: 'drive9_shared_db_pool_total{namespace="mem9",db_pool_id="1",db_pool_uuid="pool-failed",tidbcloud_org_id="97",status="failed",pod="drive9-1"}' values: '1+0x12' - series: 'drive9_shared_db_pool_capacity{namespace="mem9",db_pool_id="2",db_pool_uuid="pool-capacity",tidbcloud_org_id="97",type="used",pod="drive9-0"}' values: '91+0x12' - series: 'drive9_shared_db_pool_capacity{namespace="mem9",db_pool_id="2",db_pool_uuid="pool-capacity",tidbcloud_org_id="97",type="used",pod="drive9-1"}' values: '91+0x12' - series: 'drive9_shared_db_pool_capacity{namespace="mem9",db_pool_id="2",db_pool_uuid="pool-capacity",tidbcloud_org_id="97",type="hard_max",pod="drive9-0"}' values: '100+0x12' - series: 'drive9_shared_db_pool_capacity{namespace="mem9",db_pool_id="2",db_pool_uuid="pool-capacity",tidbcloud_org_id="97",type="hard_max",pod="drive9-1"}' values: '100+0x12' - series: 'drive9_shared_db_pool_capacity{namespace="mem9",db_pool_id="3",db_pool_uuid="pool-boundary",tidbcloud_org_id="97",type="used"}' values: '90+0x12' - series: 'drive9_shared_db_pool_capacity{namespace="mem9",db_pool_id="3",db_pool_uuid="pool-boundary",tidbcloud_org_id="97",type="hard_max"}' values: '100+0x12' - series: 'drive9_shared_db_pool_capacity{namespace="mem9",db_pool_id="4",db_pool_uuid="pool-unlimited",tidbcloud_org_id="97",type="used"}' values: '100+0x12' - series: 'drive9_shared_db_pool_capacity{namespace="mem9",db_pool_id="4",db_pool_uuid="pool-unlimited",tidbcloud_org_id="97",type="hard_max"}' values: '0+0x12' alert_rule_test: - eval_time: 4m alertname: Drive9SharedDBPoolWaitRateHigh exp_alerts: [] - eval_time: 10m alertname: Drive9SharedDBPoolWaitRateHigh exp_alerts: - exp_labels: namespace: mem9 db_pool_uuid: pool-hot tidbcloud_org_id: "97" severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 shared DB pool wait rate is high" description: "Drive9 (ns=mem9) shared DB pool pool-hot / TiDB org 97 has been waiting on sql.DB connections at more than 1 wait/sec for 5 minutes. All tenants placed on this physical DB pool may be affected. Current value: 1.2000000000000002 waits/sec" - eval_time: 18m alertname: Drive9SharedDBPoolWaitRateHigh exp_alerts: [] - eval_time: 4m alertname: Drive9SharedDBPoolProvisionFailed exp_alerts: [] - eval_time: 6m alertname: Drive9SharedDBPoolProvisionFailed exp_alerts: - exp_labels: namespace: mem9 db_pool_id: "1" db_pool_uuid: pool-failed tidbcloud_org_id: "97" severity: critical response_class: page component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 shared DB pool provisioning has failed" description: "Drive9 (ns=mem9) shared DB pool pool-failed (local ID 1) / TiDB org 97 has remained failed for 5 minutes. Reserved tenants may remain provisioning until replacement capacity is created." - eval_time: 4m alertname: Drive9SharedDBPoolHardCapacityHigh exp_alerts: [] - eval_time: 6m alertname: Drive9SharedDBPoolHardCapacityHigh exp_alerts: - exp_labels: namespace: mem9 db_pool_id: "2" db_pool_uuid: pool-capacity tidbcloud_org_id: "97" severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 shared DB pool is near hard capacity" description: "Drive9 (ns=mem9) shared DB pool pool-capacity (local ID 2) / TiDB org 97 has exceeded 90% of its emergency hard tenant capacity for 5 minutes. Provision or recover another physical shared DB pool before this pool becomes unable to accept tenants. Current usage: 91%" - interval: 1m input_series: - series: 'drive9_db_operation_duration_seconds_bucket{namespace="mem9",role="shared",le="1"}' values: '0+0x12' - series: 'drive9_db_operation_duration_seconds_bucket{namespace="mem9",role="shared",le="2.5"}' values: '0+0x12' - series: 'drive9_db_operation_duration_seconds_bucket{namespace="mem9",role="shared",le="5"}' values: '0+10x12' - series: 'drive9_db_operation_duration_seconds_bucket{namespace="mem9",role="shared",le="+Inf"}' values: '0+10x12' alert_rule_test: - eval_time: 4m alertname: Drive9SharedDBOperationP99High exp_alerts: [] - eval_time: 10m alertname: Drive9SharedDBOperationP99High exp_alerts: - exp_labels: namespace: mem9 severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 shared DB operation p99 latency is high" description: "Drive9 (ns=mem9) aggregate shared DB p99 operation latency exceeded 2.5 seconds over the past 5 minutes. Multiple tenants on one or more physical shared DB pools may be affected; use shared DB pool wait metrics and logs for attribution. Current value: 4.975s" - interval: 1m input_series: - series: 'drive9_db_operations_total{namespace="mem9",role="shared",result="error"}' values: '0+20x12' - series: 'drive9_db_operations_total{namespace="mem9",role="shared",result="ok"}' values: '0+80x12' - series: 'drive9_db_operations_total{namespace="mem9",role="user",result="error"}' values: '0+100x12' alert_rule_test: - eval_time: 4m alertname: Drive9SharedDBOperationErrorRateHigh exp_alerts: [] - eval_time: 10m alertname: Drive9SharedDBOperationErrorRateHigh exp_alerts: - exp_labels: namespace: mem9 severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 shared DB operation error ratio is high" description: "Drive9 (ns=mem9) aggregate shared DB operation errors exceeded 10% over 5 minutes. Multiple tenants on one or more physical shared DB pools may be affected; use shared DB pool wait metrics and logs for attribution. Current value: 20%" - interval: 5m input_series: # Stuck handle: zero tenant-backend refs for the whole window. The idle # reaper should have closed it after the 30m shared-handle TTL. - series: 'drive9_shared_db_pool_cache_handles{namespace="mem9",pod="drive9-0",db_pool_id="1",db_pool_uuid="pool-stuck",tidbcloud_org_id="97"}' values: '1+0x11' - series: 'drive9_shared_db_pool_cache_tenants{namespace="mem9",pod="drive9-0",db_pool_id="1",db_pool_uuid="pool-stuck",tidbcloud_org_id="97"}' values: '0+0x11' # Same physical pool on another pod with live refs: stays silent and # must not cross labels with the stuck handle above. - series: 'drive9_shared_db_pool_cache_handles{namespace="mem9",pod="drive9-1",db_pool_id="1",db_pool_uuid="pool-stuck",tidbcloud_org_id="97"}' values: '1+0x11' - series: 'drive9_shared_db_pool_cache_tenants{namespace="mem9",pod="drive9-1",db_pool_id="1",db_pool_uuid="pool-stuck",tidbcloud_org_id="97"}' values: '1+0x11' # Re-referenced at 30m: pending must reset and never reach 45m. - series: 'drive9_shared_db_pool_cache_handles{namespace="mem9",pod="drive9-0",db_pool_id="2",db_pool_uuid="pool-rewarmed",tidbcloud_org_id="97"}' values: '1+0x11' - series: 'drive9_shared_db_pool_cache_tenants{namespace="mem9",pod="drive9-0",db_pool_id="2",db_pool_uuid="pool-rewarmed",tidbcloud_org_id="97"}' values: '0+0x6 1+0x5' # Reaped at 30m as designed: series disappear, never fires. - series: 'drive9_shared_db_pool_cache_handles{namespace="mem9",pod="drive9-0",db_pool_id="3",db_pool_uuid="pool-reaped",tidbcloud_org_id="97"}' values: '1+0x6' - series: 'drive9_shared_db_pool_cache_tenants{namespace="mem9",pod="drive9-0",db_pool_id="3",db_pool_uuid="pool-reaped",tidbcloud_org_id="97"}' values: '0+0x6' alert_rule_test: - eval_time: 40m alertname: Drive9SharedDBPoolCacheHandleStuck exp_alerts: [] - eval_time: 55m alertname: Drive9SharedDBPoolCacheHandleStuck exp_alerts: - exp_labels: namespace: mem9 pod: drive9-0 db_pool_id: "1" db_pool_uuid: pool-stuck tidbcloud_org_id: "97" severity: warning response_class: logging component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 shared DB handle idle past reaper TTL" description: "Drive9 (ns=mem9) pod drive9-0 has kept a cached sql.DB handle on shared DB pool pool-stuck (local ID 1) / TiDB org 97 with zero tenant-backend refs for 45 minutes, past the 30-minute shared-handle idle TTL. The idle reaper should have closed it; this indicates a stuck reaper or a pinned handle." - interval: 1m input_series: - series: 'drive9_service_operations_total{namespace="mem9",component="event_bus",operation="publish",result="error",pod="drive9-0"}' values: '0+1x12' - series: 'drive9_sse_resets_sent_total{namespace="mem9",reason="seq_too_old",tenant_id="tenant-sse",tidbcloud_org_id="97",pod="drive9-0"}' values: '0+1x12' - series: 'drive9_service_operations_total{namespace="mem9",component="event_bus",operation="retention_sweep",result="error",pod="drive9-0"}' values: '0+1x12' - series: 'drive9_feature_enabled{namespace="mem9",feature="vault",pod="drive9-0"}' values: '1+0x12' - series: 'drive9_module_up{namespace="mem9",module="vault",pod="drive9-0"}' values: '0+0x12' # A disabled pod exports module_up=0 but must not page. - series: 'drive9_feature_enabled{namespace="mem9",feature="vault",pod="drive9-disabled"}' values: '0+0x12' - series: 'drive9_module_up{namespace="mem9",module="vault",pod="drive9-disabled"}' values: '0+0x12' alert_rule_test: - eval_time: 10m alertname: Drive9SSEPersistenceFailureRateHigh exp_alerts: - exp_labels: namespace: mem9 severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 SSE event persistence failure ratio is high" description: "Drive9 (ns=mem9) failed to persist one or more durable SSE events in the past 15 minutes. Cross-pod replay may lose precise invalidations. Current value: 10 failures" - eval_time: 11m alertname: Drive9SSEReplayMissesHigh exp_alerts: - exp_labels: namespace: mem9 severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 SSE replay misses are high" description: "Drive9 (ns=mem9) SSE durable replay reported more than 5 seq_too_old misses in 15 minutes. Current value: 11 misses" - eval_time: 10m alertname: Drive9SSERetentionSweepFailures exp_alerts: - exp_labels: namespace: mem9 severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 SSE event retention sweep is failing" description: "Drive9 (ns=mem9) SSE event retention sweep recorded failures within the past 15 minutes. Current value: 10 failures" - eval_time: 10m alertname: Drive9VaultUnavailable exp_alerts: - exp_labels: namespace: mem9 pod: drive9-0 severity: critical response_class: page component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 vault module is unavailable" description: "Drive9 (ns=mem9) vault module has been unavailable for 5 minutes while the service is still exporting metrics." - interval: 1m input_series: - series: 'container_memory_working_set_bytes{namespace="drive9-tidbcloud",pod="drive9-server-memory",container="drive9-server"}' values: '920+0x18' - series: 'kube_pod_container_resource_limits{namespace="drive9-tidbcloud",pod="drive9-server-memory",container="drive9-server",resource="memory",unit="byte"}' values: '1000+0x18' - series: 'container_memory_working_set_bytes{namespace="drive9-tidbcloud",pod="drive9-server-memory-ok",container="drive9-server"}' values: '850+0x18' - series: 'kube_pod_container_resource_limits{namespace="drive9-tidbcloud",pod="drive9-server-memory-ok",container="drive9-server",resource="memory",unit="byte"}' values: '1000+0x18' - series: 'kube_pod_status_scheduled{namespace="drive9-tidbcloud",pod="drive9-server-pending",condition="false"}' values: '1+0x18' alert_rule_test: - eval_time: 16m alertname: Drive9PodMemoryLimitHigh exp_alerts: - exp_labels: namespace: drive9-tidbcloud pod: drive9-server-memory severity: critical response_class: page component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 pod memory usage is high" description: "Drive9 (ns=drive9-tidbcloud) pod drive9-server-memory has used more than 85% of its memory limit for 15 minutes; it is at risk of OOMKilled. Current value: 92%" - eval_time: 11m alertname: Drive9PodUnschedulable exp_alerts: - exp_labels: namespace: drive9-tidbcloud pod: drive9-server-pending severity: critical response_class: page component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 pod is unschedulable" description: "Drive9 pod drive9-tidbcloud/drive9-server-pending has remained unschedulable for 10 minutes; check node capacity, scheduling constraints, and resource requests." # Sustained stale claimable backlog: claimable>0 and age>3600 for 15m. - interval: 1m input_series: - series: 'drive9_file_gc_claimable{namespace="mem9",tenant_id="t-gc-1",tidbcloud_org_id="org-1"}' values: '10x20' - series: 'drive9_file_gc_oldest_claimable_age_seconds{namespace="mem9",tenant_id="t-gc-1",tidbcloud_org_id="org-1"}' values: '7200x20' alert_rule_test: - eval_time: 16m alertname: Drive9FileGCClaimableBacklogStale exp_alerts: - exp_labels: namespace: mem9 tenant_id: t-gc-1 tidbcloud_org_id: org-1 severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 file GC claimable backlog is stale for a tenant" description: "Drive9 (ns=mem9) tenant t-gc-1 / TiDB org org-1 has claimable file_gc tasks (claimable > 0) whose oldest available_at is older than 1 hour for 15 minutes. Storage quota may stay inflated after deletes. This signal is emitted only for already warm/observed tenant backends on a kick/drain path, not a cold-tenant fleet scan. Check tenant_worker_slot_busy / slot_held_long / kick_delayed logs and file_gc_tasks. Current oldest age: 2h 0m 0s" # Strict age boundary: age == 3600 must not alert (expression uses > 3600). - interval: 1m input_series: - series: 'drive9_file_gc_claimable{namespace="mem9",tenant_id="t-gc-boundary",tidbcloud_org_id="org-b"}' values: '10x20' - series: 'drive9_file_gc_oldest_claimable_age_seconds{namespace="mem9",tenant_id="t-gc-boundary",tidbcloud_org_id="org-b"}' values: '3600x20' alert_rule_test: - eval_time: 16m alertname: Drive9FileGCClaimableBacklogStale exp_alerts: [] # Zero backlog: no claimable series (sparse gauges omit zero) must not alert. - interval: 1m input_series: - series: 'drive9_http_requests_total{namespace="mem9",method="GET",route="/healthz",status="200"}' values: '1+0x20' alert_rule_test: - eval_time: 16m alertname: Drive9FileGCClaimableBacklogStale exp_alerts: [] # Active drain: hit_cap series present must suppress stale-backlog paging. - interval: 1m input_series: - series: 'drive9_file_gc_claimable{namespace="mem9",tenant_id="t-gc-drain",tidbcloud_org_id="org-d"}' values: '10000x20' - series: 'drive9_file_gc_oldest_claimable_age_seconds{namespace="mem9",tenant_id="t-gc-drain",tidbcloud_org_id="org-d"}' values: '7200x20' - series: 'drive9_file_gc_drain_hit_cap{namespace="mem9",tenant_id="t-gc-drain",tidbcloud_org_id="org-d"}' values: '1x20' alert_rule_test: - eval_time: 16m alertname: Drive9FileGCClaimableBacklogStale exp_alerts: [] # Pure file_gc/sched/slot_busy_error is multi-worker slot contention, not a # hard worker failure — must not page the umbrella background-worker alerts. - interval: 1m input_series: - series: 'drive9_service_operations_total{namespace="mem9",component="file_gc",operation="sched",result="slot_busy_error",pod="drive9-0"}' values: '0+1x20' - series: 'drive9_tenant_operation_failures_total{namespace="mem9",component="file_gc",operation="sched",result="slot_busy_error",tenant_id="t-slot",tidbcloud_org_id="org-s"}' values: '0+1x20' alert_rule_test: - eval_time: 16m alertname: Drive9BackgroundWorkerErrorsHigh exp_alerts: [] - eval_time: 16m alertname: Drive9BackgroundWorkerTenantErrorsHigh exp_alerts: [] # Real file_gc process errors must still fire the umbrella alerts. - interval: 1m input_series: - series: 'drive9_service_operations_total{namespace="mem9",component="file_gc",operation="process",result="error",pod="drive9-0"}' values: '0+1x20' - series: 'drive9_tenant_operation_failures_total{namespace="mem9",component="file_gc",operation="process",result="error",tenant_id="t-err",tidbcloud_org_id="org-e"}' values: '0+1x20' alert_rule_test: - eval_time: 16m alertname: Drive9BackgroundWorkerErrorsHigh exp_alerts: - exp_labels: namespace: mem9 worker: file_gc operation: process severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 background worker is failing" description: "Drive9 (ns=mem9) background worker file_gc/process recorded more than 5 errors or deadline timeouts over 15 minutes. This includes unscoped control-plane paths such as expiry sweeps and mutation replay listing. Current value: 15 failures" - eval_time: 16m alertname: Drive9BackgroundWorkerTenantErrorsHigh exp_alerts: - exp_labels: namespace: mem9 worker: file_gc operation: process tenant_id: t-err tidbcloud_org_id: org-e severity: major response_class: ticket component: drive9 oncall_team: drive9 exp_annotations: summary: "Drive9 background worker is failing for a tenant" description: "Drive9 (ns=mem9) background worker file_gc/process for tenant t-err / TiDB org org-e recorded more than 5 attributable failures over 15 minutes. Current value: 15 failures"