{ "schemaVersion": "auto-oncall-scenario-rule-evaluation-v1", "frozenAt": "2026-08-06T13:40:00+08:00", "experiments": [ { "scenarioId": "query-execution", "rule": "Treat TopSQL, Slow Query, and decoded-plan findings as query evidence by\ndefault. Promote a specific SQL or plan to this scenario's causal mechanism\nonly when it is leading or materially dominant on the affected entity, aligns\nwith the incident timeline, and explains the observed resource or latency\npath. Pair DML execution count with affected rows, scanned/processed rows, or\nwrite volume; count alone is not workload magnitude. A top-ranked digest from\nanother instance or a wider window remains correlation only.", "cases": [ { "issueKey": "TCID-13001", "role": "target", "environment": "cloud", "question": "Which query raised the load and delayed the DDL in the reported window?", "evidence": [ "The incident window is 2026-08-04 14:58:50 to 15:01:36 UTC.", "Two TiDB instances recorded 7,135 slow-query rows; TiDB CPU and RSS peaked at 15:01:20.", "KV, coprocessor, PD wait, TS wait, and backoff accumulated in the same window.", "The available TopSQL candidates have no digest, instance, or execution link to the target DDL.", "No slow-query row matching the DDL prefix was found in the incident window.", "The supplied memory alert fired after the target window." ] }, { "issueKey": "TCID-12986", "role": "target", "environment": "cloud", "question": "Did the identified aggregation query cause the TiKV-13 CPU increase?", "evidence": [ "The aggregation query is slow on plan 1DD1366F because an index join estimated 52 rows but read about 333,779 rows.", "The same plan served the query before and during the incident; there was no mid-incident plan switch.", "The query explains about one third of the CPU growth on TiKV-13, not the full node differential.", "Scale-in rebalancing overlaps the increase, but current Region and Leader counts do not show TiKV-13 as an outlier.", "Other workload responsible for the remaining CPU growth has not been identified." ] }, { "issueKey": "APID-29251", "role": "control", "environment": "op", "question": "What caused the TiKV hotspot?", "evidence": [ "ANALYZE TABLE on the affected instance consumed about 384.9 CPU-seconds, around six times the second SQL.", "About 98.8 percent of its CPU was concentrated in 09:52:10 to 09:52:30, aligned with the TiKV spike.", "It peaked at about 16.8 cores and scanned about 39,000 records per second.", "The node CPU returned to about 6.4 cores after the task ended." ] }, { "issueKey": "TCID-12987", "role": "control", "environment": "cloud", "question": "Why did the previously fast query become slow?", "evidence": [ "The SQL text and plan stayed the same, but average keys read per execution rose from about 555 to about 1,229.", "Slow executions were concentrated in a 19-second burst; each slow sample processed about 50,000 keys.", "TiKV suspend time dominated process and wait time during those samples.", "When the business re-enabled this query pattern, the latency symptom reproduced; application optimization resolved the issue." ] } ] }, { "scenarioId": "memory-oom", "rule": "Do not classify OOM from warning volume alone. Require actual heap/quota\npressure, an OOM kill, restart, allocation failure, or query failure. Keep\nmemory exhaustion distinct from GC or allocator pressure: GC pause, frequency,\nor duration needs its own aligned evidence and remains contributing or\nsecondary unless it explains the user-visible CPU/latency path better than the\nheap, workload, or capacity mechanism.", "cases": [ { "issueKey": "ST-2847", "role": "target", "environment": "cloud", "question": "Does the TiKV memory alert indicate an OOM risk?", "evidence": [ "One node stayed around 72 to 73 percent memory and crossed a 70 percent alert threshold.", "TiKV RSS stayed roughly 39.4 to 43.1 GiB without a spike or persistent rise.", "RocksDB block cache accounted for about 26.3 GiB per instance and was stable.", "CPU, disk, QPS, Leader distribution, and Region trend showed no aligned failure signal.", "There was no OOM kill, restart, eviction, or allocation failure." ] }, { "issueKey": "ST-2865", "role": "target", "environment": "cloud", "question": "What can be concluded from the greater-than-80-percent TiKV memory warning?", "evidence": [ "Several TiKV pods exceeded 80 percent and the highest reached about 82.94 percent; cluster average was about 76 percent.", "No OOM, restart, or eviction occurred in the incident window.", "Allocator memory was high, but the available aggregate metric could not bind it to a specific pod or distinguish active allocations, retention, or fragmentation.", "RocksDB memtable and reader memory were low; Region-count skew was only a candidate contributor." ] }, { "issueKey": "ST-2862", "role": "control", "environment": "cloud", "question": "Why did the SQL fail with ERROR 8175?", "evidence": [ "The failed execution reached 10.008 GiB against a 10 GiB tidb_mem_quota_query limit.", "The same query later succeeded at 9.926 GiB.", "The IndexHashJoin path did not support spill for this workload and the failed execution was cancelled.", "The two executions differed by only about 84 MiB near the quota boundary." ] }, { "issueKey": "APID-29041", "role": "control", "environment": "op", "question": "Is this an actual TiDB OOM event?", "evidence": [ "A self-managed TiDB server process terminated with an OOM condition.", "The node continued writing oom_record entries after the event.", "TopSQL did not show a dominant query candidate.", "The supplied evidence proves the OOM event but not the workload or allocator root cause." ] } ] }, { "scenarioId": "capacity-resource-pressure", "rule": "When node count, worker count, quota, or effective capacity fell first and\nexplains the before/after differential, keep downstream TopSQL, GC, and hotspot\nsignals as contributing or secondary unless one workload independently\nexplains most of that differential. Conversely, a capacity alert is not\nprimary when a leading workload or IO mechanism explains the pressure better.", "cases": [ { "issueKey": "TCID-12926", "role": "target", "environment": "cloud", "question": "Did scale-in cause the coprocessor wait increase?", "evidence": [ "Scale-in ran from about 20:15 through the early 20:40s, overlapping much of the wait increase.", "The later investigation found a hot index Leader concentrated on TiKV-29 with very high QPS.", "TiKV-29 accumulated coprocessor tasks while CPU remained below an obvious saturation threshold.", "Increasing readpool threads could delay the symptom but was assessed as mitigation, not the root fix.", "Scatter or workload-plan changes addressed the concentrated hot range." ] }, { "issueKey": "TCID-12985", "role": "target", "environment": "cloud", "question": "Was the 80,000 RCU quota too small for the latency incident?", "evidence": [ "RU usage peaked near 21,399 RU/s against an 80,000 RCU quota.", "Resource-control wait was about 0.5 ms while end-to-end latency was multiple seconds.", "One TiDB instance rose from about 0.49 to nearly 3 cores and saturated after maintenance ended and users reconnected.", "A burst of concurrent inserts and possible parent-row lock contention remained the leading workload mechanism." ] }, { "issueKey": "ST-2864", "role": "control", "environment": "cloud", "question": "Is the TiFlash 85-percent alert a capacity problem or an acute IO/apply incident?", "evidence": [ "All three TiFlash instances used about 3.3 TiB of 3.9 TiB, around 85 percent.", "Append-log, apply-log, raft-engine write and sync latency stayed low, with no pending snapshot apply.", "There was no aligned user-visible latency incident.", "The immediate operational issue was remaining disk headroom and growth rate." ] }, { "issueKey": "APID-29277", "role": "control", "environment": "op", "question": "What is the primary risk while TiKV disks are 93 to 95 percent used?", "evidence": [ "Three TiKV stores were around 93 to 95 percent disk use and beyond PD's 0.8 low-space ratio.", "TiCDC lag grew to about 13 to 14 hours while leader rebalancing and compaction were active.", "A fourth store was added, but existing stores temporarily fell to about 5 percent free during redistribution.", "The required control was disk headroom and safe rebalance completion before adding workload." ] } ] }, { "scenarioId": "tikv-storage-io", "rule": "Distinguish workload-induced IO saturation from an independent infrastructure\nor storage-engine failure. If scan/write volume rises first and explains the IO\nboundary, keep storage as contributing; if disk, throttle, engine, or\nwrite-stall evidence starts earlier or independently explains the result, keep\nstorage primary. Tiny residual throttle values are not sufficient by\nthemselves.", "cases": [ { "issueKey": "TCID-12948", "role": "target", "environment": "cloud", "question": "Was roughly 100 MiB/s per TiKV a disk limit causing slow COMMIT?", "evidence": [ "Disk load was high but retained headroom; CPU and MiB/s did not move as a simple saturation pair.", "Prewrite work was high and TopSQL showed one-row INSERTs plus UPDATEs that rewrote an indexed deleted_at column.", "Batching inserts and removing unnecessary index maintenance reduced TiKV load and improved the load test.", "No independent disk fault, throttle event, or write stall preceded the workload increase." ] }, { "issueKey": "APID-29259", "role": "target", "environment": "op", "question": "What drove the TiKV CPU and commit-latency spike?", "evidence": [ "Write bytes on one store rose about 5.4 times in the aligned window.", "sched_worker and RocksDB accounted for about 72 percent of the CPU increase.", "Scheduler latch wait concentrated on the same store and explained commit P99 better than aggregate CPU.", "The evidence did not separate transactions contending on the same keys from slower storage extending latch hold time.", "The aggregate COMMIT digest and prev_stmt could not identify the originating transaction." ] }, { "issueKey": "APID-29211", "role": "control", "environment": "op", "question": "What caused the TiKV slow-store and PD leader-change alerts?", "evidence": [ "The TiKV data disk reached about 100 percent IO utilization and write latency reached hundreds of milliseconds at only tens of MiB/s.", "PD WAL fdatasync reached about four seconds and the PD leader changed in the same window.", "Host disk latency and internal WAL/raft signals aligned on the affected devices.", "There was no workload-volume increase sufficient to explain the device behavior." ] }, { "issueKey": "APID-29298", "role": "control", "environment": "op", "question": "Why did Slow Score rise about 50 minutes after inspected duration?", "evidence": [ "Inspected duration includes wait and commit phases that Slow Score does not count.", "Slow Score rises from qualifying raft-disk timeouts over repeated inspect ticks, not from a fixed 50-minute timer.", "Disk write latency later rose to about 3 to 6 ms with WAL-related tails.", "The metrics prove a local write-path delay but cannot prove a cloud disk or host fault, nor reconstruct why each earlier sample was not scored." ] } ] }, { "scenarioId": "tiflash-io-apply", "rule": "Distinguish workload-induced TiFlash IO from an independent disk, engine, or\nreplica failure by ordering and dominance. Query/read volume that rises first\nmay be the primary mechanism; a disk or apply-path signal that starts earlier\nor independently explains the user-visible result keeps this scenario primary.", "cases": [ { "issueKey": "APID-29265", "role": "target", "environment": "op", "question": "What caused the periodic TiFlash raft-wait-index spikes?", "evidence": [ "Wait-index and apply-wait rose periodically while append-log, apply-log, and raft-message-wait stayed in milliseconds.", "A query began choosing TiFlash because the optimizer overestimated an index scan at about 8.61 million rows while the actual index result was zero.", "After a binding forced the existing TiKV index plan, wait-index, upstream latency, and MPP duration stopped showing the periodic spikes for multiple days.", "The post-change observation is consistent with the query plan as the upstream trigger." ] }, { "issueKey": "APID-12588", "role": "target", "environment": "op", "question": "Was the raft-wait-index alert an independent TiFlash disk failure?", "evidence": [ "Concentrated reads of several tables raised TiFlash disk read traffic and latency, followed by apply-log delay and the alert.", "The relevant join lacked a selective useridx-leading index and scanned excessive data.", "Adding the proposed index and removing the forced TiFlash path produced a large performance improvement.", "No independent disk failure preceded the query load." ] }, { "issueKey": "APID-12208", "role": "control", "environment": "op", "question": "What caused the raft-wait-index alert across TiFlash nodes?", "evidence": [ "Six TiFlash hosts showed a broad, simultaneous disk-latency jitter.", "Only whichever node carried TiFlash work at that moment emitted the alert, but the host-level jitter was cluster-wide.", "No single query or replica-flow increase was shown to precede and explain the broad device behavior." ] }, { "issueKey": "ST-2766", "role": "control", "environment": "cloud", "question": "Why were several TiFlash MPP queries slow?", "evidence": [ "Only the TiFlash-4 to TiFlash-6 MPP tunnels logged connection failures in the incident window.", "Other TiFlash node pairs did not show the error and ordinary disk/apply metrics did not identify an IO bottleneck.", "Task accumulation on TiFlash-4 was downstream of the failed inter-node stream.", "A separate SQL in the ticket was slow because TiKV was a better plan, but it was not evidence of a disk failure." ] } ] } ] }