# TCOC Auto Replayer - Case Library
#
# Each case represents a real incident that has been manually postmortem'd.
# The replayer runs each case through Investibook and scores the result
# against the expected.md baseline.
#
# See docs/sops/tcoc-postmortem.md for the SOP.
# See cases/TEMPLATE.md for the postmortem template.

cases:
  - issue_id: "O11Y-2328599"
    prompt: "Use tidb-clinic-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "v8.5.4"
    tags: ["expensive-query", "slow-query", "hotspot", "missing-index"]
    expected_root_cause: "Missing index on dh_message_setting_ext.tag_id causing TiKV-19 hotspot"
    date_added: "2026-04-07"
    reviewer: "lingtao.li"

  - issue_id: "O11Y-2179140"
    prompt: "Use tidb-clinic-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "premium"
    tidb_version: "v8.5.4"
    tags: ["p999-latency", "lock-contention", "hotspot", "pessimistic-lock", "high-freq-update"]
    expected_root_cause: "High-frequency UPDATE on t_merchant_online causing pessimistic lock contention hotspot on single TiKV instance"
    date_added: "2026-04-07"
    reviewer: "lingtao.li"

  - issue_id: "TCOC-5011"
    prompt: "Use tidb-clinic-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tags: ["coprocessor-worker", "capacity-failure", "cascade", "pod-restart", "OOM"]
    expected_root_cause: "3/4 coprocessor TiKV workers went offline simultaneously, single remaining worker overloaded and OOM'd repeatedly, causing 50min full cluster outage"
    date_added: "2026-04-10"
    reviewer: "lingtao.li"

  - issue_id: "TCOC-5019"
    prompt: "Use tidb-clinic-diagnosis to analyze this Premium cluster P999 SQL Duration spike. Establish the incident window, collect basic evidence, inspect TiDB GC/heap profiling, correlate the heap profile with slow queries and known TiDB issues, and provide root cause, evidence chain, and recommendations."
    cluster_type: "premium"
    tidb_version: "v8.5.4"
    tags: ["p999-latency", "gc-pressure", "heap-profile", "splitKeyRangesByBuckets", "region-bucket", "known-bug"]
    expected_root_cause: "TiDB v8.5.4 known bug #66096 in splitKeyRangesByBuckets triggered by heavy IndexLookUp/range workloads, causing KeyRanges.Slice heap bloat, GC pressure, and P999 SQL Duration spikes"
    date_added: "2026-04-15"
    reviewer: "huyibin"

  - issue_id: "TCOC-4964"
    prompt: "Use tidb-clinic-diagnosis to analyze this TiDB periodic CPU spike. Establish the incident window, inspect continuous profiling for db-tidb-0/1/2/5 peak periods, correlate profiling stacks with slow logs/TopSQL/logs/events, and provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "v8.5.4"
    tags: ["cpu-spike", "continuous-profiling", "plan-cache-miss", "large-in-list", "optimizer", "gc-pressure"]
    expected_root_cause: "Large IN-list prepared statements skip plan cache, forcing repeated optimizer generateNewPlan/range derivation, causing heavy malloc/GC and periodic TiDB CPU spikes on db-tidb-0/1/2/5"
    date_added: "2026-04-15"
    reviewer: "huyibin"

  - issue_id: "O11Y-2298309"
    prompt: "Use tidb-clinic-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "v8.5.2"
    tags: ["panic", "pod-restart", "known-bug", "kill-query", "memory-tracker"]
    expected_root_cause: "TiDB v8.5.2 known bug (GitHub #63956): KILL QUERY on idle transaction triggers panic in memory.Tracker.Consume() during rollback cleanup"
    date_added: "2026-04-07"
    reviewer: "claude-code"

  - issue_id: "O11Y-2392287"
    prompt: "Use tidb-clinic-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "v8.5.4"
    tags: ["expensive-query", "auto-recovered", "jira-api-broken", "auto-diagnosis"]
    expected_root_cause: "Sustained expensive query workload (>100/min for ~1hr) causing TiKV gRPC latency spikes and NetworkLagIncrease alerts on dedicated cluster"
    date_added: "2026-04-16"
    reviewer: "claude-code"

  - issue_id: "O11Y-2429368"
    prompt: "Use tidb-clinic-diagnosis to analyze this panic alert. Find the panic log, classify it, and determine if it requires action."
    cluster_type: "dedicated"
    tidb_version: "v8.5.4"
    tags: ["panic", "recoverable-goroutine", "timeout-interrupt", "expected-behavior", "panic-classification"]
    expected_root_cause: "Recoverable goroutine panic from query timeout interrupt ([executor:3024] max_execution_time exceeded) — expected behavior, not a bug"
    date_added: "2026-04-20"
    reviewer: "claude-code"

  - issue_id: "TCOC-5140"
    prompt: "Premium cluster prod-c66-seamless-others, investigate latency spike during 2026-04-16 10:51~10:59 UTC+8."
    cluster_type: "premium"
    tidb_version: "v8.5.4"
    tags: ["latency-spike", "rolling-restart", "scale-operation", "false-oom", "prestop-hook", "sigterm"]
    expected_root_cause: "Human-initiated ScaleResourcePool (TiKV CPU upgrade) triggered K8s rolling restart with SIGTERM, causing Raft leader migration storms and latency spikes — not OOM"
    date_added: "2026-04-20"
    reviewer: "claude-code"

  - issue_id: "GH-229"
    prompt: "Investigate CPU spike on dedicated cluster 10518506906906971664 during 2026-04-08 06:00-09:30 UTC+8."
    cluster_type: "dedicated"
    tidb_version: "v8.5.4"
    tags: ["cpu-spike", "plan-cache-misdiagnosis", "prepare-exec-close", "scale-in", "node-reduction", "gc-pressure"]
    expected_root_cause: "TiDB node count reduction (27→18) increased per-node load +65%; plan cache hit=0 is expected for prepare:exec:close=1:1:1 pattern, not the root cause"
    date_added: "2026-04-20"
    reviewer: "claude-code"

  - issue_id: "O11Y-2426677"
    prompt: "Use tidb-clinic-diagnosis to analyze this expensive query alert. Find the expensive queries from TiDB logs, identify root cause, and provide recommendations."
    cluster_type: "dedicated"
    tags: ["expensive-query", "missing-index", "index-full-scan", "stats-divergence", "tikv-hotspot", "ebs-iops-throttle"]
    expected_root_cause: "Missing index on dh_account_basic.last_update_timestamp causing IndexFullScan of ~52M rows, saturating db-tikv-67/74 CPU and triggering 100+ expensive queries"
    date_added: "2026-04-20"
    reviewer: "claude-code"

  - issue_id: "GH-264"
    prompt: "Use tidb-clinic-diagnosis to analyze this expensive query alert. Find the expensive queries from TiDB logs, identify root cause, and provide recommendations."
    cluster_type: "dedicated"
    tags: ["cross-workspace-consistency", "expensive-query", "missing-index", "hot-node-attribution", "topsql-ranking"]
    expected_root_cause: "Same as O11Y-2426677 — missing index on dh_account_basic.last_update_timestamp. Cross-workspace case: WS-A reached wrong conclusion (attributed hotspot to wrong SQL), WS-B was correct."
    related_ticket: "O11Y-2426677"
    workspaces:
      - id: "lark-ws-5c83e277-065b-4f29-8c06-033756674f83"
        label: "WS-A (incorrect)"
        score: 2.4
      - id: "lark-ws-a7da5d80-9dce-456b-b947-627f49eb03df"
        label: "WS-B (correct)"
        score: 4.0
    date_added: "2026-04-21"
    reviewer: "claude-code"

  - issue_id: "O11Y-2489480"
    prompt: "Use tidb-clinic-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "unknown"
    tags: ["pod-restart", "oom-kill", "gc-surge", "tooling-gap", "loki-404", "events-access-denied", "japan-gcp"]
    expected_root_cause: "Single TiDB pod db-tidb-76 restarted due to K8s cgroup OOM Kill; GC frequency surge 6x in final 4 minutes is key signal; confirmation blocked by Loki 404 and Activity Hub Access Denied"
    date_added: "2026-05-01"
    reviewer: "claude-code"

  - issue_id: "O11Y-2530675"
    prompt: "Use tidb-clinic-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "v8.5.4"
    tags: ["latency-spike", "ebs-iops-throttle", "tikv-hotspot", "lock-contention", "wait-time", "recommendation-priority"]
    expected_root_cause: "TiKV infrastructure saturation: EBS IOPS throttle up to 40% on 5 TiKV volumes + db-tikv-9 gRPC hotspot (17.5x vs peers); query wait_time 231s >> process_time 10s confirms IO queuing not inefficient execution"
    date_added: "2026-05-08"
    reviewer: "claude-code"

  - issue_id: "TCID-12692"
    prompt: "Use tidb-clinic-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "v8.5.3"
    tags: ["cpu-saturation", "ru-exhaustion", "proxysql-traffic-shift", "kill-events", "autoscale", "tikv-hotspot", "topsql-unavailable", "japan-gcp"]
    expected_root_cause: "Cluster-wide TiDB CPU saturation (25+ nodes at 80-88%) driven by SELECT workload surge (~515K stmt/s); all resource groups exhausted RU budgets; ProxySQL traffic shift caused kill events; db-tikv-26 read hotspot is downstream effect"
    date_added: "2026-05-15"
    reviewer: "claude-code"

  - issue_id: "TCID-12704"
    prompt: "Use tidb-clinic-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "v8.5.3"
    tags: ["scale-in-during-load", "client-disconnect", "client-retry", "coupon-query", "in-clause-overload", "activity-hub", "topsql-unavailable", "japan-gcp"]
    expected_root_cause: "coupon_attached_items query with 177-range IN-clause caused TiDB CPU 75% saturation; ModifyClusterSize scale-in (34→33 nodes) at 14:37:37 JST dropped existing connections, directly causing client disconnections and retries; scale-out 33→34 at 14:50 ended incident at 14:53"
    date_added: "2026-05-19"
    reviewer: "claude-code"

  - issue_id: "TCOC-5382"
    prompt: "Use tidb-perf-diagnosis to analyze this connection timeout incident. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "v8.5.3"
    tags: ["connection-timeout", "scale-out-disruption", "ru-saturation", "sqlkiller-mass-kill", "recoverable-panic", "tikv-hotspot", "japan-gcp", "topsql-unavailable", "multi-window"]
    expected_root_cause: "Three independent fault windows: (1) scale-out 39→41 TiDB pods not yet ready → ECONNRESET; (2+3) RU saturation + TiKV db-tikv-15 hotspot drove mass queries over max_execution_time → SQLKiller kill wave → recoverable panics on db-tidb-2946/2954 → ECONNREFUSED"
    date_added: "2026-06-10"
    reviewer: "claude-code"

  - issue_id: "TCOC-5387"
    prompt: "Use tidb-clinic-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "v8.5.4-cygames"
    tags: ["tiproxy-bug", "dns-confusion", "cpu-imbalance", "zone-routing", "memory-leak", "tiproxy-restart"]
    expected_root_cause: "TiProxy bug tiproxy/issues/1172: simultaneous pod restart caused DNS confusion, stale long connections to wrong zonal owner, empty CPU metrics returned for Zone A TiDB, routing exclusion concentrating all traffic on 14/15 TiDB nodes"
    date_added: "2026-06-24"
    reviewer: "evolver-bot"

  - issue_id: "O11Y-2612493"
    prompt: "Use tidb-perf-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "unknown"
    tags: ["tikv-scale-out", "pd-region-balance", "error-1317", "query-interrupted", "ru-saturation", "japan-gcp", "topsql-unavailable", "activity-hub"]
    expected_root_cause: "6 consecutive TiKV scale-out batches (36→54 nodes) triggered PD region balance storms; migrating regions interrupted KV requests with error 1317 across 36+ TiDB nodes, driving error rate 4.5× baseline; RU exhaustion amplified failures"
    date_added: "2026-07-13"
    reviewer: "evolver-bot"

  - issue_id: "O11Y-2622673"
    prompt: "Use tidb-perf-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "unknown"
    tags: ["expensive-query", "slow-query", "multi-plan", "index-selection", "reviews-table", "epoch-bug", "mysql-cast"]
    expected_root_cause: "reviews table aggregate SQL (bc6171 digest) has 8 concurrent plan_digests; plans 2d7c47cf (avg 9.1s) and 1dd1366f (avg 6.9s) have poor index selection (total_keys >> process_keys), driving expensive query count above 500/10min threshold since 2026-07-24 07:22 UTC"
    date_added: "2026-07-27"
    reviewer: "evolver-bot"

  - issue_id: "TCID-12948"
    prompt: "Use tidb-clinic-diagnosis to analyze this write latency issue. Provide root cause, evidence chain, and recommendations."
    cluster_type: "dedicated"
    tidb_version: "unknown"
    tags: ["tikv-cpu-saturation", "high-frequency-small-transactions", "2pc-rpc-fanout", "write-latency", "batch-insert", "gcp-pd-throughput", "autochat-timeout"]
    expected_root_cause: "TiKV cluster-wide CPU saturation (13/21 nodes at 80-96%) from unbatched high-frequency small transactions (5,300 INSERT/s + 3,466 UPDATE/s), each generating individual 2PC RPC, saturating gRPC threadpool; resolve_lock latch wait P99=19.14s, storage async write P99=7.32s (100x baseline), COMMIT slow log 184,537 entries avg 2.67s; disk write stall=0, CPU is primary bottleneck"
    date_added: "2026-07-29"
    reviewer: "evolver-bot"

  - issue_id: "TCID-12985"
    prompt: "Use tidb-perf-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "premium"
    tidb_version: "unknown"
    tags: ["tidb-cpu-saturation", "rcu-exhaustion", "goroutine-queuing", "workload-spike", "resource-control", "slow-query-wait-time", "scale-up"]
    expected_root_cause: "TiDB CPU saturation on two 3-core-quota nodes (wxc2zi 99%, 35bplo 81%) caused by ~10x workload spike at 14:01 CST; slow query wait_time=1.81s >> process_time=0.03s with TiKV cop 1.54ms confirms TiDB-layer goroutine queuing; RCU available_ru=0 is a consequence of CPU saturation (request_p90=0.5ms proves RCU throttle overhead is negligible); resolved by ScaleInstanceTiDB 4C→8C completed 14:52"
    date_added: "2026-08-07"
    reviewer: "evolver-bot"

  - issue_id: "TCOC-5841"
    prompt: "Use tidb-perf-diagnosis to analyze this alert. Provide root cause, evidence chain, and recommendations."
    cluster_type: "essential"
    tidb_version: "unknown"
    tags: ["cdc-resolvets-lag", "noisy-neighbor", "essential-shared", "ticdc-replication-worker", "changefeed", "orphaned-locks", "tidb-restart"]
    expected_root_cause: "Neighbor cluster's changefeed on the same shared pool consumed TiCDC replication worker slots, blocking keyspace 126719's changefeed from advancing min_resolved_ts; resolveTs lag persisted >2 minutes — noisy-neighbor pattern in Essential shared TiCDC layer (no tenant-level worker isolation)"
    date_added: "2026-08-24"
    reviewer: "evolver-bot"

  - issue_id: "TCOC-5870"
    prompt: "Use tidb-clinic-diagnosis to analyze this issue. Provide root cause, evidence chain, and recommendations."
    cluster_type: "essential"
    tidb_version: "v8.5.3"
    tags: ["changefeed-stuck-creating", "resource-limitation", "rcu-saturation", "essential-provisioning-gate", "ticdc-pod-not-created", "activity-hub", "operation-logs"]
    expected_root_cause: "Essential cluster provisionedRCU=maxRCU=50000 (sysbench workload saturating RCU) triggered ResourceLimitation; platform provisioning gate blocked TiCDC K8s pod allocation; changefeed stuck indefinitely in 'creating' state — 0 ticdc operations in manager logs confirms pod never attempted"
    date_added: "2026-08-28"
    reviewer: "evolver-bot"